{"meta":{"start_date":"2026-07-07","end_date":"2026-07-07","start_at":"2026-07-07T00:00:00Z","end_at":"2026-07-08T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T23:08:56Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":5,"failed_runs":5,"raw_occurrences":17,"matched_occurrences":17,"runs_affected":5},"rows":[{"environment":"stg","failure_pattern_id":"941cbd5c6e3f3d8ed572893a689b2dfded75f418444a1a0d01baa9fc9e30d9c0","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-f62dnq, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-f62dnq\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":11,"runs_affected":3,"failed_runs":3,"run_impact_percent":60,"anchor_occurrences":11,"after_last_push_count":11,"trend_counts":[0,0,0,0,0,0,11],"trend_range":"2026-07-01..2026-07-07 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-08","2026-06-15","2026-06-22","2026-06-29"],"prior_runs_affected":44,"prior_last_seen_at":"2026-07-05T23:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002206708\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-f62dnq, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-f62dnq\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002206708\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-f62dnq, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-f62dnq\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-bnbl2n\" for cluster \"cluster-nightly-4-21-bnbl2n\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0029876e0\u003e: \n    failed to create NodePool np-bnbl2n, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-bnbl2n in resource group rg-nightly-4-21-8f8wq5, error: failed waiting for nodepool=\"np-bnbl2n\" for cluster \"cluster-nightly-4-21-bnbl2n\" in resourcegroup=\"rg-nightly-4-21-8f8wq5\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-bnbl2n\" for cluster \"cluster-nightly-4-21-bnbl2n\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0029876e0\u003e: \n    failed to create NodePool np-bnbl2n, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-bnbl2n in resource group rg-nightly-4-21-8f8wq5, error: failed waiting for nodepool=\"np-bnbl2n\" for cluster \"cluster-nightly-4-21-bnbl2n\" in resourcegroup=\"rg-nightly-4-21-8f8wq5\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:136]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002992210\u003e: \n    failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-zrmw7h, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-zrmw7h\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:136]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002992210\u003e: \n    failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-zrmw7h, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-zrmw7h\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2074629625115316224","occurred_at":"2026-07-07T23:00:22Z","signature_id":"54b19eeed4f9dc2163b78247efe9307a478a952c5c556d62caf319505b42ff71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2074629625115316224","occurred_at":"2026-07-07T23:00:22Z","signature_id":"68931080a24ff43a9414a110ce773f3107df03a2bcd1b171a0a4fcb569a071fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2074629625115316224","occurred_at":"2026-07-07T23:00:22Z","signature_id":"836bde506547278c8b2ff0bcf666441fb6c7bdf15caf255bfbf2caa1c5d2edfa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2074629625115316224","occurred_at":"2026-07-07T23:00:22Z","signature_id":"befe4e9dd09f034219c21592d63d3782b3da4198afefa6b373327f56b9ac7b37","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2074629625115316224","occurred_at":"2026-07-07T23:00:22Z","signature_id":"d9db319efa072d18d7db3449338be9c0ee86226eb29e41537de7e862cad8f168","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074498265448976384","occurred_at":"2026-07-07T14:18:23Z","signature_id":"6a27a3ff172892988d8041a835febb68f7ba8e70b23b54cbe412aa6c5e0ba844","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074498265448976384","occurred_at":"2026-07-07T14:18:23Z","signature_id":"9ee752460202bdf100a5a9e370b1cedff777e738999bf341330a611956744b23","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074498265448976384","occurred_at":"2026-07-07T14:18:23Z","signature_id":"e5c960d6cef78720975897912cf0bd6e4a5522461573203e44f0a57d5a91d281","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2074312532230672384","occurred_at":"2026-07-07T02:00:21Z","signature_id":"13e37252cfa3991f65a89275a096ae2f75800e61b81244d824cc6328161af31e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2074312532230672384","occurred_at":"2026-07-07T02:00:21Z","signature_id":"5747b7bdd2590746db1aaa71304a104d6eb67f0865b8f78c12d14d905036b1d1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2074312532230672384","occurred_at":"2026-07-07T02:00:21Z","signature_id":"630f5eaf2e7cf71df7d7720ff934091c2b90e30fcb5f7cfab0e9be633bc3c598","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-22-z2nlc2, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-z2nlc2 in resource group rg-candidate-4-22-sd2944, error: failed waiting for cluster=\"cluster-candidate-4-22-z2nlc2\" in resourcegroup=\"rg-candidate-4-22-sd2944\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-z2nlc2 in resource group rg-candidate-4-22-sd2944, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":80,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-07-01..2026-07-07 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-08","2026-06-15","2026-06-22","2026-06-29"],"prior_runs_affected":55,"prior_last_seen_at":"2026-07-05T21:44:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-4-23-xqs86j/cluster-nightly-4-23-j4cd8k should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00064a0f0\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-j4cd8k, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-j4cd8k in resource group rg-nightly-4-23-xqs86j, error: failed waiting for cluster=\"cluster-nightly-4-23-j4cd8k\" in resourcegroup=\"rg-nightly-4-23-xqs86j\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-j4cd8k in resource group rg-nightly-4-23-xqs86j, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-4-23-xqs86j/cluster-nightly-4-23-j4cd8k should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00064a0f0\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-j4cd8k, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-j4cd8k in resource group rg-nightly-4-23-xqs86j, error: failed waiting for cluster=\"cluster-nightly-4-23-j4cd8k\" in resourcegroup=\"rg-nightly-4-23-xqs86j\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-j4cd8k in resource group rg-nightly-4-23-xqs86j, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-sd2944/cluster-candidate-4-22-z2nlc2 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009e23f0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-z2nlc2, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-z2nlc2 in resource group rg-candidate-4-22-sd2944, error: failed waiting for cluster=\"cluster-candidate-4-22-z2nlc2\" in resourcegroup=\"rg-candidate-4-22-sd2944\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-z2nlc2 in resource group rg-candidate-4-22-sd2944, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-sd2944/cluster-candidate-4-22-z2nlc2 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009e23f0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-z2nlc2, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-z2nlc2 in resource group rg-candidate-4-22-sd2944, error: failed waiting for cluster=\"cluster-candidate-4-22-z2nlc2\" in resourcegroup=\"rg-candidate-4-22-sd2944\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-z2nlc2 in resource group rg-candidate-4-22-sd2944, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-wb46hj/cluster-candidate-5-0-89fnng should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e80e40\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-89fnng, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-89fnng in resource group rg-candidate-5-0-wb46hj, error: failed waiting for cluster=\"cluster-candidate-5-0-89fnng\" in resourcegroup=\"rg-candidate-5-0-wb46hj\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-89fnng in resource group rg-candidate-5-0-wb46hj, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-wb46hj/cluster-candidate-5-0-89fnng should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e80e40\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-89fnng, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-89fnng in resource group rg-candidate-5-0-wb46hj, error: failed waiting for cluster=\"cluster-candidate-5-0-89fnng\" in resourcegroup=\"rg-candidate-5-0-wb46hj\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-89fnng in resource group rg-candidate-5-0-wb46hj, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2074629625115316224","occurred_at":"2026-07-07T23:00:22Z","signature_id":"c1871f1359496de7c8c68f624252e86f082916988ef91b156a558b290c3b3b10","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074498265448976384","occurred_at":"2026-07-07T14:18:23Z","signature_id":"64fc37eb018cab706e9805a8823c90d351e6288cc2f5d461d55c1ebb1f11d53d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074392124802994176","occurred_at":"2026-07-07T07:16:32Z","signature_id":"0d415ca605dce99007bac15caf772c5573c31ee3a2ecd554a96241d2a8850b16","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074351837527412736","occurred_at":"2026-07-07T04:33:47Z","signature_id":"cf11b4f2a4442bf22d15e78f1220f7114d4645d596bfe28381ff18165647ae62","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"9d76b8574a881c48ca814c73a5f78cfd9c34fbf0c2d12d09c9f9b6e856adedcf","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,1,0,0,0,0,1],"trend_range":"2026-07-01..2026-07-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-29"],"prior_runs_affected":2,"prior_last_seen_at":"2026-07-02T23:00:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013f8420\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-5rpjlm, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-5rpjlm\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013f8420\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-5rpjlm, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-5rpjlm\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074498265448976384","occurred_at":"2026-07-07T14:18:23Z","signature_id":"4b7652f80ed03439468763a73437788cecbc7ae7fcac59db1e39b07e9dba0231","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-01..2026-07-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-22"],"prior_runs_affected":4,"prior_last_seen_at":"2026-06-26T17:38:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-xjp2wq\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f18280\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-2x2t8.apps.aro.a3a2u9b0b6c9e9l.h96i.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-xjp2wq\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f18280\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-2x2t8.apps.aro.a3a2u9b0b6c9e9l.h96i.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2074312532230672384","occurred_at":"2026-07-07T02:00:21Z","signature_id":"a10eac73a6b07673dd3aa6609874c4b7abf5ad5f2c04de66d009e78d4b2554bd","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}