{"meta":{"start_date":"2026-04-03","end_date":"2026-04-03","start_at":"2026-04-03T00:00:00Z","end_at":"2026-04-04T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T18:27:55Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":2,"failed_runs":2,"raw_occurrences":3,"matched_occurrences":3,"runs_affected":2},"rows":[{"environment":"int","failure_pattern_id":"a880b7dbbe8686372b606e0212fd3742bc75c3c0a9c545bae9481b243ed0f55b","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-s2z7xg, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-s2z7xg\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-28..2026-04-03 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-02","2026-03-16","2026-03-23"],"prior_runs_affected":5,"prior_last_seen_at":"2026-03-26T22:57:56Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000b37470\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-s2z7xg, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-s2z7xg\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-s2z7xg, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-s2z7xg\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000340210\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-s2z7xg\",\n            },\n            \u003c*fmt.wrapError | 0xc000f90d40\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-s2z7xg\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000b37470\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-s2z7xg, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-s2z7xg\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-s2z7xg, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-s2z7xg\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000340210\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-s2z7xg\",\n            },\n            \u003c*fmt.wrapError | 0xc000f90d40\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-s2z7xg\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2040151865483595776","occurred_at":"2026-04-03T19:38:03Z","signature_id":"981e7e47da1bfeee5b247671b92d57bc0ef5ad3b922f6e266e65f71764e7befd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"2fbd0ba2d30f9973767b159fc6c9ed931483d2933c7b233f5ee7ab3c3bb59219","failure_pattern":"failed to cleanup resource group: failed waiting for resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: failed waiting for resourcegroup=\"rg-negative-tests-pgqhpg\" to finish deleting: context canceled","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-28..2026-04-03 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001238348\u003e: \n    failed to cleanup resource group: failed waiting for resourcegroup=\"rg-negative-tests-pgqhpg\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc001245340\u003e{\n                msg: \"failed to cleanup resource group: failed waiting for resourcegroup=\\\"rg-negative-tests-pgqhpg\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc001245320\u003e{\n                    msg: \"failed waiting for resourcegroup=\\\"rg-negative-tests-pgqhpg\\\" to finish deleting: context canceled\",\n                    err: \u003c*errors.errorString | 0xbee2c60\u003e{\n                        s: \"context canceled\",\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001238348\u003e: \n    failed to cleanup resource group: failed waiting for resourcegroup=\"rg-negative-tests-pgqhpg\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc001245340\u003e{\n                msg: \"failed to cleanup resource group: failed waiting for resourcegroup=\\\"rg-negative-tests-pgqhpg\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc001245320\u003e{\n                    msg: \"failed waiting for resourcegroup=\\\"rg-negative-tests-pgqhpg\\\" to finish deleting: context canceled\",\n                    err: \u003c*errors.errorString | 0xbee2c60\u003e{\n                        s: \"context canceled\",\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2040151865483595776","occurred_at":"2026-04-03T19:38:03Z","signature_id":"f57aee9fca6b2350b7846f1c4f00cfbab62495b7b412bc8d6ea3268b3b1f38e1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"310e24b2db3f0c4d638044465004cd03afcce87baf5784081066c35dfd42e8e6","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"e2e-cilium-96sc9p\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-28..2026-04-03 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:186]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001b03140\u003e: \n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"e2e-cilium-96sc9p\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed waiting for nodepool=\\\"cilium-np\\\" for cluster \\\"cilium-cluster\\\" in resourcegroup=\\\"e2e-cilium-96sc9p\\\" to finish creating: context deadline exceeded\",\n        err: \u003ccontext.deadlineExceededError\u003e{},\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:186]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001b03140\u003e: \n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"e2e-cilium-96sc9p\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed waiting for nodepool=\\\"cilium-np\\\" for cluster \\\"cilium-cluster\\\" in resourcegroup=\\\"e2e-cilium-96sc9p\\\" to finish creating: context deadline exceeded\",\n        err: \u003ccontext.deadlineExceededError\u003e{},\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2040191555695808512","occurred_at":"2026-04-03T22:15:46Z","signature_id":"9f05ebc75fba9e8de9b4613e8897ff9757b3222c878bae066d56be68c58c2937","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}