{"meta":{"start_date":"2026-07-06","end_date":"2026-07-12","start_at":"2026-07-06T00:00:00Z","end_at":"2026-07-13T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T19:24:09Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":28,"failed_runs":20,"raw_occurrences":87,"matched_occurrences":85,"runs_affected":18},"rows":[{"environment":"stg","failure_pattern_id":"941cbd5c6e3f3d8ed572893a689b2dfded75f418444a1a0d01baa9fc9e30d9c0","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-8xvhv8, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-8xvhv8\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":30,"runs_affected":10,"failed_runs":10,"run_impact_percent":35.714285714285715,"anchor_occurrences":30,"after_last_push_count":30,"trend_counts":[0,13,11,5,0,1,0,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-08","2026-06-15","2026-06-22","2026-06-29"],"prior_runs_affected":47,"prior_last_seen_at":"2026-07-05T23:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:151]: failed to create node pool npupgrade-4-20 with version 4.20.28\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002592660\u003e: \n    failed to create NodePool npupgrade-4-20, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-20 in resource group rg-np-version-upgrade-wjg86x-d529tv, error: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-wjg86x\" in resourcegroup=\"rg-np-version-upgrade-wjg86x-d529tv\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:151]: failed to create node pool npupgrade-4-20 with version 4.20.28\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002592660\u003e: \n    failed to create NodePool npupgrade-4-20, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-20 in resource group rg-np-version-upgrade-wjg86x-d529tv, error: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-wjg86x\" in resourcegroup=\"rg-np-version-upgrade-wjg86x-d529tv\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:136]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0022a58c0\u003e: \n    failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-nl4k78, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-nl4k78\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:136]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0022a58c0\u003e: \n    failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-nl4k78, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-nl4k78\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:136]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0027151a0\u003e: \n    failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-plcsc4, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-plcsc4\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:136]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0027151a0\u003e: \n    failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-plcsc4, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-plcsc4\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2075483182756532224","occurred_at":"2026-07-10T07:32:05Z","signature_id":"6bb8dda8c692800e3c2aff0d9195c986d7755ebe84bf5179be9d33f3d9f973a3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074756168701448192","occurred_at":"2026-07-08T07:23:12Z","signature_id":"4daaf25ae963162750be6450f2e716f639f8e3d5e5b56b13008481876242a3b7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2074674924420599808","occurred_at":"2026-07-08T02:00:22Z","signature_id":"5e4b9778df247146ec61c9e2daed0235b98322ae21edfa780dd497a34c5311f4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2074674924420599808","occurred_at":"2026-07-08T02:00:22Z","signature_id":"6c8fec4a6a816c322e7528c55d042c7ecf1763a77f555573954012392e99ea4f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2074674924420599808","occurred_at":"2026-07-08T02:00:22Z","signature_id":"d7940a396114e68786ac0835598ca44e55c1205ab59af74c223a43881b141e52","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2074674924420599808","occurred_at":"2026-07-08T02:00:22Z","signature_id":"f984235edbb0aa43c358e008e64ea3f2c991301a6546f53e5ef13f5e797dc8f2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2074629625115316224","occurred_at":"2026-07-07T23:00:22Z","signature_id":"54b19eeed4f9dc2163b78247efe9307a478a952c5c556d62caf319505b42ff71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2074629625115316224","occurred_at":"2026-07-07T23:00:22Z","signature_id":"68931080a24ff43a9414a110ce773f3107df03a2bcd1b171a0a4fcb569a071fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2074629625115316224","occurred_at":"2026-07-07T23:00:22Z","signature_id":"836bde506547278c8b2ff0bcf666441fb6c7bdf15caf255bfbf2caa1c5d2edfa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2074629625115316224","occurred_at":"2026-07-07T23:00:22Z","signature_id":"befe4e9dd09f034219c21592d63d3782b3da4198afefa6b373327f56b9ac7b37","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2074629625115316224","occurred_at":"2026-07-07T23:00:22Z","signature_id":"d9db319efa072d18d7db3449338be9c0ee86226eb29e41537de7e862cad8f168","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074498265448976384","occurred_at":"2026-07-07T14:18:23Z","signature_id":"6a27a3ff172892988d8041a835febb68f7ba8e70b23b54cbe412aa6c5e0ba844","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074498265448976384","occurred_at":"2026-07-07T14:18:23Z","signature_id":"9ee752460202bdf100a5a9e370b1cedff777e738999bf341330a611956744b23","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074498265448976384","occurred_at":"2026-07-07T14:18:23Z","signature_id":"e5c960d6cef78720975897912cf0bd6e4a5522461573203e44f0a57d5a91d281","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2074312532230672384","occurred_at":"2026-07-07T02:00:21Z","signature_id":"13e37252cfa3991f65a89275a096ae2f75800e61b81244d824cc6328161af31e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2074312532230672384","occurred_at":"2026-07-07T02:00:21Z","signature_id":"5747b7bdd2590746db1aaa71304a104d6eb67f0865b8f78c12d14d905036b1d1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2074312532230672384","occurred_at":"2026-07-07T02:00:21Z","signature_id":"630f5eaf2e7cf71df7d7720ff934091c2b90e30fcb5f7cfab0e9be633bc3c598","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2074267232510152704","occurred_at":"2026-07-06T23:00:20Z","signature_id":"9162b5a6c9a764beb0efc649b31d31d35c573b3714fc50eb149b26173e83a845","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2074267232510152704","occurred_at":"2026-07-06T23:00:20Z","signature_id":"a7e9bffa7cb75aa6b057f6092da5e22a9d39b29123e9c89a4394ba1c92b5b4af","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2074267232510152704","occurred_at":"2026-07-06T23:00:20Z","signature_id":"acde3be073994ec6dc65b96efce888e00a7413a90304cc77aa097d38d8939e2a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074102247611961344","occurred_at":"2026-07-06T12:04:45Z","signature_id":"1f4e6f53e782a4fefff53adef224b3c6c0a9b08920f4e705ae5a2619787110e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074102247611961344","occurred_at":"2026-07-06T12:04:45Z","signature_id":"55875c963f292ed1ab6042fb7437a6306f4475f26eb9ac708bafb637836ebda1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074102247611961344","occurred_at":"2026-07-06T12:04:45Z","signature_id":"e2df6e218b36f9d2b6808799506a3ea7dc21124d84d9d028f1d1c634269d1aa0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074027709368897536","occurred_at":"2026-07-06T07:08:34Z","signature_id":"04e84a6a64c4d736660c343b28e8153b5f8836bb984dde570555f0a00a8636bd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074027709368897536","occurred_at":"2026-07-06T07:08:34Z","signature_id":"58b40796ac7188482ac377c14880cd5e92cd2e95044c2fc3d01c827f2bd6216b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2073950128116338688","occurred_at":"2026-07-06T02:00:17Z","signature_id":"c9bd6595c74af94968462a8ff937bd2fb1a97880a3bd406fdd3287791b0b2651","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2073950128116338688","occurred_at":"2026-07-06T02:00:17Z","signature_id":"e269e9a3066f2b5cc944ce9eec4c39c14b8352fa048863df2b5236c2f5b92774","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2073950128116338688","occurred_at":"2026-07-06T02:00:17Z","signature_id":"f052f6d4ae727c1734ad26512ca743f54fb08b224cd0bec941b2ed02b3360c5b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2073950128116338688","occurred_at":"2026-07-06T02:00:17Z","signature_id":"f3a7a4947faef948e7a13657de4e110122dd2ece4c540956de58aea835a7b1fa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2073950128116338688","occurred_at":"2026-07-06T02:00:17Z","signature_id":"fff54a59ba89bab2608477d548b6111c38cf063101a480e6ffb31fe151ef5829","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-6hdcwz, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-6hdcwz in resource group rg-candidate-4-21-gtv29c, error: failed waiting for cluster=\"cluster-candidate-4-21-6hdcwz\" in resourcegroup=\"rg-candidate-4-21-gtv29c\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-6hdcwz in resource group rg-candidate-4-21-gtv29c, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":18,"runs_affected":11,"failed_runs":11,"run_impact_percent":39.285714285714285,"anchor_occurrences":18,"after_last_push_count":18,"trend_counts":[0,4,4,2,0,8,0,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-08","2026-06-15","2026-06-22","2026-06-29"],"prior_runs_affected":59,"prior_last_seen_at":"2026-07-05T21:44:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000accde0\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-92v58x, error: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-92v58x\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-92v58x, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000accde0\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-92v58x, error: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-92v58x\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-92v58x, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-21-gtv29c/cluster-candidate-4-21-6hdcwz should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001116420\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-6hdcwz, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-6hdcwz in resource group rg-candidate-4-21-gtv29c, error: failed waiting for cluster=\"cluster-candidate-4-21-6hdcwz\" in resourcegroup=\"rg-candidate-4-21-gtv29c\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-6hdcwz in resource group rg-candidate-4-21-gtv29c, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-21-gtv29c/cluster-candidate-4-21-6hdcwz should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001116420\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-6hdcwz, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-6hdcwz in resource group rg-candidate-4-21-gtv29c, error: failed waiting for cluster=\"cluster-candidate-4-21-6hdcwz\" in resourcegroup=\"rg-candidate-4-21-gtv29c\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-6hdcwz in resource group rg-candidate-4-21-gtv29c, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:101]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0028aa5a0\u003e: \n    failed to create HCP cluster np-autoscale-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-whrdjm, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-whrdjm\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-whrdjm, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:101]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0028aa5a0\u003e: \n    failed to create HCP cluster np-autoscale-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-whrdjm, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-whrdjm\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-whrdjm, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2075625536775786496","occurred_at":"2026-07-10T16:57:45Z","signature_id":"09e43dc5e5e0445b72f0340be397ea83e802441cf7b41581c4eeb8fa6b70a3a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2075483182756532224","occurred_at":"2026-07-10T07:32:05Z","signature_id":"45f561d0cf9362d242ff0034dba71fa15fbd96d0a24576adceafc787311d4285","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2075399707978371072","occurred_at":"2026-07-10T02:00:24Z","signature_id":"146eecd7d8879c02a0dcfaaf8c796493a667f941eeaecbb68f189f6101d577d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2075399707978371072","occurred_at":"2026-07-10T02:00:24Z","signature_id":"1bc8566a51895e10d0236fed6f73e36fec9c73fd8064d59823a94a06d6297f92","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2075399707978371072","occurred_at":"2026-07-10T02:00:24Z","signature_id":"475b6d2b8821dc27667c60cadb73b1506ee49509fc8274b0235dc14f55a1d690","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2075399707978371072","occurred_at":"2026-07-10T02:00:24Z","signature_id":"b0a8b1a3e056748c18066b4208ff06e0877b7e70bdeb6e1d9f9f92b40910e732","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2075399707978371072","occurred_at":"2026-07-10T02:00:24Z","signature_id":"d1f09c4f69432a903a3f68c33513a4f6ba47410d2d37f06bd8a9a9a01a416865","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2075399707978371072","occurred_at":"2026-07-10T02:00:24Z","signature_id":"ff167fe9b2523becc3e8a415291cd834584a122a5dfeb2e9008200656af99a1f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2074674924420599808","occurred_at":"2026-07-08T02:00:22Z","signature_id":"08f6e598006757bf07b9b60c73605a315d7be6f550f947ff949ce6aab3b4558e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2074674924420599808","occurred_at":"2026-07-08T02:00:22Z","signature_id":"1fe31d263420bfc81794ac0763076c0eb478194bb5d40bbd0b30292cbef0a8c8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2074629625115316224","occurred_at":"2026-07-07T23:00:22Z","signature_id":"c1871f1359496de7c8c68f624252e86f082916988ef91b156a558b290c3b3b10","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074498265448976384","occurred_at":"2026-07-07T14:18:23Z","signature_id":"64fc37eb018cab706e9805a8823c90d351e6288cc2f5d461d55c1ebb1f11d53d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074392124802994176","occurred_at":"2026-07-07T07:16:32Z","signature_id":"0d415ca605dce99007bac15caf772c5573c31ee3a2ecd554a96241d2a8850b16","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074351837527412736","occurred_at":"2026-07-07T04:33:47Z","signature_id":"cf11b4f2a4442bf22d15e78f1220f7114d4645d596bfe28381ff18165647ae62","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2074267232510152704","occurred_at":"2026-07-06T23:00:20Z","signature_id":"0af8dda48579619e35b906c52445ee547366c75eb93309e51434c8eb39f9e858","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2074267232510152704","occurred_at":"2026-07-06T23:00:20Z","signature_id":"30c32fc47ab53c6040878b2ef6ffe4eb8ccbed30e1757d45f1cf8eea4f729719","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074102247611961344","occurred_at":"2026-07-06T12:04:45Z","signature_id":"5bbbe312e60242a0080ced2d50d08db529bde225105803f3ec6b9ccc500a1d4f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2073950128116338688","occurred_at":"2026-07-06T02:00:17Z","signature_id":"3fcc931cd205b8fd060b407d7a5ed6339fbe8803c4d8cd3b2c8c164a8c89beed","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"ec1b1336058eef4e52617c8415c248a74c85b9b6bac45ee9d8d64d61efb0b226","failure_pattern":"failed to get latest install version for candidate channel: query candidate graph for \u003cocp-channel\u003e: Get \"\u003curl\u003e\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","search_query":"fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:179]: failed to get latest install version for candidate channel: query candidate graph for candidate-4.20: Get \"https://api.openshift.com/api/upgrades_info/v1/graph?channel=candidate-4.20\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":10,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":10,"after_last_push_count":10,"trend_counts":[0,10,0,0,0,0,0,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:179]: failed to get latest install version for candidate channel: query candidate graph for candidate-4.20: Get \"https://api.openshift.com/api/upgrades_info/v1/graph?channel=candidate-4.20\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:179]: failed to get latest install version for candidate channel: query candidate graph for candidate-4.20: Get \"https://api.openshift.com/api/upgrades_info/v1/graph?channel=candidate-4.20\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074027709368897536","occurred_at":"2026-07-06T07:08:34Z","signature_id":"a5ecc4176a5d6acfdaab61e279d004441af852c5ecbbe8365a6be5b677958fbb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074027709368897536","occurred_at":"2026-07-06T07:08:34Z","signature_id":"a5ecc4176a5d6acfdaab61e279d004441af852c5ecbbe8365a6be5b677958fbb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074027709368897536","occurred_at":"2026-07-06T07:08:34Z","signature_id":"a5ecc4176a5d6acfdaab61e279d004441af852c5ecbbe8365a6be5b677958fbb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074027709368897536","occurred_at":"2026-07-06T07:08:34Z","signature_id":"a5ecc4176a5d6acfdaab61e279d004441af852c5ecbbe8365a6be5b677958fbb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074027709368897536","occurred_at":"2026-07-06T07:08:34Z","signature_id":"a5ecc4176a5d6acfdaab61e279d004441af852c5ecbbe8365a6be5b677958fbb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074027709368897536","occurred_at":"2026-07-06T07:08:34Z","signature_id":"a5ecc4176a5d6acfdaab61e279d004441af852c5ecbbe8365a6be5b677958fbb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074027709368897536","occurred_at":"2026-07-06T07:08:34Z","signature_id":"a5ecc4176a5d6acfdaab61e279d004441af852c5ecbbe8365a6be5b677958fbb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074027709368897536","occurred_at":"2026-07-06T07:08:34Z","signature_id":"a5ecc4176a5d6acfdaab61e279d004441af852c5ecbbe8365a6be5b677958fbb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074027709368897536","occurred_at":"2026-07-06T07:08:34Z","signature_id":"a5ecc4176a5d6acfdaab61e279d004441af852c5ecbbe8365a6be5b677958fbb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074027709368897536","occurred_at":"2026-07-06T07:08:34Z","signature_id":"a5ecc4176a5d6acfdaab61e279d004441af852c5ecbbe8365a6be5b677958fbb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"4e83fe1c3bb6daff1be6935129ccd1a1c67e021531f7a464980385c362ccc372","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-candidate-4-22-p27992\", caused by: failed waiting for hcpCluster=\"cluster-candidate-4-22-bf67pt\" in resourcegroup=\"rg-candidate-4-22-p27992\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-4-22-bf67pt in resource group rg-candidate-4-22-p27992, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-candidate-4-22-bf67pt\" in resourcegroup=\"rg-candidate-4-22-p27992\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-4-22-bf67pt in resource group rg-candidate-4-22-p27992, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":8,"runs_affected":3,"failed_runs":3,"run_impact_percent":10.714285714285714,"anchor_occurrences":8,"after_last_push_count":8,"trend_counts":[0,0,0,1,0,5,0,2],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc00108a450\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-candidate-4-22-p27992\", caused by: failed waiting for hcpCluster=\"cluster-candidate-4-22-bf67pt\" in resourcegroup=\"rg-candidate-4-22-p27992\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-4-22-bf67pt in resource group rg-candidate-4-22-p27992, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-candidate-4-22-bf67pt\" in resourcegroup=\"rg-candidate-4-22-p27992\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-4-22-bf67pt in resource group rg-candidate-4-22-p27992, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc00108a450\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-candidate-4-22-p27992\", caused by: failed waiting for hcpCluster=\"cluster-candidate-4-22-bf67pt\" in resourcegroup=\"rg-candidate-4-22-p27992\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-4-22-bf67pt in resource group rg-candidate-4-22-p27992, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-candidate-4-22-bf67pt\" in resourcegroup=\"rg-candidate-4-22-p27992\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-4-22-bf67pt in resource group rg-candidate-4-22-p27992, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc001b34240\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-labels-taints-n2l8g2\", caused by: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-n2l8g2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-n2l8g2, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-n2l8g2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-n2l8g2, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc001b34240\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-labels-taints-n2l8g2\", caused by: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-n2l8g2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-n2l8g2, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-n2l8g2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-n2l8g2, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc000fb8498\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-skip-minor-7wfv2w-wrn8zb\", caused by: failed waiting for hcpCluster=\"np-skip-minor-7wfv2w\" in resourcegroup=\"rg-np-skip-minor-7wfv2w-wrn8zb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-7wfv2w in resource group rg-np-skip-minor-7wfv2w-wrn8zb, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-skip-minor-7wfv2w\" in resourcegroup=\"rg-np-skip-minor-7wfv2w-wrn8zb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-7wfv2w in resource group rg-np-skip-minor-7wfv2w-wrn8zb, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc000fb8498\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-skip-minor-7wfv2w-wrn8zb\", caused by: failed waiting for hcpCluster=\"np-skip-minor-7wfv2w\" in resourcegroup=\"rg-np-skip-minor-7wfv2w-wrn8zb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-7wfv2w in resource group rg-np-skip-minor-7wfv2w-wrn8zb, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-skip-minor-7wfv2w\" in resourcegroup=\"rg-np-skip-minor-7wfv2w-wrn8zb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-7wfv2w in resource group rg-np-skip-minor-7wfv2w-wrn8zb, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2076105588294553600","occurred_at":"2026-07-12T00:45:19Z","signature_id":"89d831ccf72982f39cc5f6ca2e54099b1d26c1bf2196c5591db6867b190d7b70","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2076105588294553600","occurred_at":"2026-07-12T00:45:19Z","signature_id":"a18f0e5dd08f60d5164e679f2c87187f311909570648924226560465b75246ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2075399707978371072","occurred_at":"2026-07-10T02:00:24Z","signature_id":"018b0a1c4bd3045bb50eb178378d1b4f96b95ceb3c412d18259816f3e0c20be5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2075399707978371072","occurred_at":"2026-07-10T02:00:24Z","signature_id":"5b6bdaefe211726ccaceedaa8d736c980ab6fd082307a01ae0c2d9419a8f0fa2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2075399707978371072","occurred_at":"2026-07-10T02:00:24Z","signature_id":"78a28aab0bf5d26c5b8c57395a9df867b496f883558f493a51bf9cea76f569dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2075399707978371072","occurred_at":"2026-07-10T02:00:24Z","signature_id":"95f39c0cd4a5412b87fd0d7b3c53a477f93b096686f4aa282d15a87e56b78a04","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2075399707978371072","occurred_at":"2026-07-10T02:00:24Z","signature_id":"9c5925d3d493788544cd3d27ab950d91b4c1c638cf0c50a9a7e19cbe7c0469cc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2074992017066168320","occurred_at":"2026-07-08T23:00:22Z","signature_id":"8622233b5b960ff67491d61a552637dc6c781ba4b726ddb2ac908be6dbb32960","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"04dd451119ec8b9089e43737c0d54b3cdaac42f33d585329b15deac499ef745d","failure_pattern":"after 4 attempts: ResponseFailed: unexpected HTTP status: 503 Service Unavailable","search_query":"after 4 attempts: ResponseFailed: unexpected HTTP status: 503 Service Unavailable","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,4,0,0,0,0,0,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:65]: failed to determine z-stream install version for minor 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d62860\u003e: \n    after 4 attempts: ResponseFailed: unexpected HTTP status: 503 Service Unavailable\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:65]: failed to determine z-stream install version for minor 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d62860\u003e: \n    after 4 attempts: ResponseFailed: unexpected HTTP status: 503 Service Unavailable\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:400]: failed to get latest version for nodepool minor 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008085c0\u003e: \n    after 4 attempts: ResponseFailed: unexpected HTTP status: 503 Service Unavailable\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:400]: failed to get latest version for nodepool minor 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008085c0\u003e: \n    after 4 attempts: ResponseFailed: unexpected HTTP status: 503 Service Unavailable\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:536]: failed to get versions for minor 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e1c460\u003e: \n    after 4 attempts: ResponseFailed: unexpected HTTP status: 503 Service Unavailable\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:536]: failed to get versions for minor 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e1c460\u003e: \n    after 4 attempts: ResponseFailed: unexpected HTTP status: 503 Service Unavailable\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074027709368897536","occurred_at":"2026-07-06T07:08:34Z","signature_id":"8cbfdd595f071eefa23ec82da4a7fb925f96dbc973eeccee234718b0d84dc472","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074027709368897536","occurred_at":"2026-07-06T07:08:34Z","signature_id":"ab0d846160ff7871cf487fc33b70d420bef85f7eaabc37dbfa99a349989323c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074027709368897536","occurred_at":"2026-07-06T07:08:34Z","signature_id":"c794fae31e90aa48b663fbe2224bf6582078c37c20cf43404a9d95569617eb62","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074027709368897536","occurred_at":"2026-07-06T07:08:34Z","signature_id":"da6d0f30455a49a34ff64f929f8cdbf9fa2059aeab318d7701e4816f452b381a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":10.714285714285714,"anchor_occurrences":3,"after_last_push_count":6,"trend_counts":[0,0,1,0,0,1,1,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-22"],"prior_runs_affected":6,"prior_last_seen_at":"2026-06-26T17:38:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:194]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0025541f8\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-k7x5x.apps.aro.cilium-cluster.omy1.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:194]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0025541f8\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-k7x5x.apps.aro.cilium-cluster.omy1.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-vcwc6l\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f82580\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-4lc9p.apps.aro.c9l5k9x0w5n8t1j.s25a.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-vcwc6l\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f82580\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-4lc9p.apps.aro.c9l5k9x0w5n8t1j.s25a.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-xjp2wq\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f18280\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-2x2t8.apps.aro.a3a2u9b0b6c9e9l.h96i.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-xjp2wq\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f18280\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-2x2t8.apps.aro.a3a2u9b0b6c9e9l.h96i.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2075762116777742336","occurred_at":"2026-07-11T02:00:29Z","signature_id":"fb98bc59891c85d7e7fb0fd8f216cb7711f16cb6372e6fb6f2bd239ccf8f99b3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2075483182756532224","occurred_at":"2026-07-10T07:32:05Z","signature_id":"f39aaf9c1303e3523ca9ba0de5dfd4ec6ed102b70870a2554a300bc1784de1f2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2074312532230672384","occurred_at":"2026-07-07T02:00:21Z","signature_id":"a10eac73a6b07673dd3aa6609874c4b7abf5ad5f2c04de66d009e78d4b2554bd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"4135b8ce49ef106a1089ff0710166e0d2c1d3ab607b56071ca50a7703cf12147","failure_pattern":"failed to get latest install version for candidate channel: after 4 attempts: query candidate graph for \u003cocp-channel\u003e returned 503 Service Unavailable: \u003chtml\u003e","search_query":"failure","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,3,0,0,0,0,0,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:179]: failed to get latest install version for candidate channel: after 4 attempts: query candidate graph for candidate-4.20 returned 503 Service Unavailable: \u003chtml\u003e\r\n  \u003chead\u003e\r\n    \u003cmeta name=\"viewport\" content=\"width=device-width, initial-scale=1\"\u003e\r\n\r\n    \u003cstyle type=\"text/css\"\u003e\r\n      body {\r\n        font-family: \"Helvetica Neue\", Helvetica, Arial, sans-serif;\r\n        line-height: 1.66666667;\r\n        font-size: 16px;\r\n        color: #333;\r\n        background-color: #fff;\r\n        margin: 2em 1em;\r\n      }\r\n      h1 {\r\n        font-size: 28px;\r\n        font-weight: 400;\r\n      }\r\n      p {\r\n        margin: 0 0 10px;\r\n      }\r\n      .alert.alert-info {\r\n        background-color: #F0F0F0;\r\n        margin-top: 30px;\r\n        padding: 30px;\r\n      }\r\n      .alert p {\r\n        padding-left: 35px;\r\n      }\r\n      ul {\r\n        padding-left: 51px;\r\n        position: relative;\r\n      }\r\n      li {\r\n        font-size: 14px;\r\n        margin-bottom: 1em;\r\n      }\r\n      p.info {\r\n        position: relative;\r\n        font-size: 20px;\r\n      }\r\n      p.info:before, p.info:after {\r\n        content: \"\";\r\n        left: 0;\r\n        position: absolute;\r\n        top: 0;\r\n      }\r\n      p.info:before {\r\n        background: #0066CC;\r\n        border-radius: 16px;\r\n        color: #fff;\r\n        content: \"i\";\r\n        font: bold 16px/24px serif;\r\n        height: 24px;\r\n        left: 0px;\r\n        text-align: center;\r\n        top: 4px;\r\n        width: 24px;\r\n      }\r\n\r\n      @media (min-width: 768px) {\r\n        body {\r\n          margin: 6em;\r\n        }\r\n      }\r\n    \u003c/style\u003e\r\n  \u003c/head\u003e\r\n  \u003cbody\u003e\r\n    \u003cdiv\u003e\r\n      \u003ch1\u003eApplication is not available\u003c/h1\u003e\r\n      \u003cp\u003eThe application is currently not serving requests at this endpoint. It may not have been started or is still starting.\u003c/p\u003e\r\n\r\n      \u003cdiv class=\"alert alert-info\"\u003e\r\n        \u003cp class=\"info\"\u003e\r\n          Possible reasons you are seeing this page:\r\n        \u003c/p\u003e\r\n        \u003cul\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eThe host doesn't exist.\u003c/strong\u003e\r\n            Make sure the hostname was typed correctly and that a route matching this hostname exists.\r\n          \u003c/li\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eThe host exists, but doesn't have a matching path.\u003c/strong\u003e\r\n            Check if the URL path was typed correctly and that the route was created using the desired path.\r\n          \u003c/li\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eRoute and path matches, but all pods are down.\u003c/strong\u003e\r\n            Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.\r\n          \u003c/li\u003e\r\n        \u003c/ul\u003e\r\n      \u003c/div\u003e\r\n    \u003c/div\u003e\r\n  \u003c/body\u003e\r\n\u003c/html\u003e\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:179]: failed to get latest install version for candidate channel: after 4 attempts: query candidate graph for candidate-4.20 returned 503 Service Unavailable: \u003chtml\u003e\r\n  \u003chead\u003e\r\n    \u003cmeta name=\"viewport\" content=\"width=device-width, initial-scale=1\"\u003e\r\n\r\n    \u003cstyle type=\"text/css\"\u003e\r\n      body {\r\n        font-family: \"Helvetica Neue\", Helvetica, Arial, sans-serif;\r\n        line-height: 1.66666667;\r\n        font-size: 16px;\r\n        color: #333;\r\n        background-color: #fff;\r\n        margin: 2em 1em;\r\n      }\r\n      h1 {\r\n        font-size: 28px;\r\n        font-weight: 400;\r\n      }\r\n      p {\r\n        margin: 0 0 10px;\r\n      }\r\n      .alert.alert-info {\r\n        background-color: #F0F0F0;\r\n        margin-top: 30px;\r\n        padding: 30px;\r\n      }\r\n      .alert p {\r\n        padding-left: 35px;\r\n      }\r\n      ul {\r\n        padding-left: 51px;\r\n        position: relative;\r\n      }\r\n      li {\r\n        font-size: 14px;\r\n        margin-bottom: 1em;\r\n      }\r\n      p.info {\r\n        position: relative;\r\n        font-size: 20px;\r\n      }\r\n      p.info:before, p.info:after {\r\n        content: \"\";\r\n        left: 0;\r\n        position: absolute;\r\n        top: 0;\r\n      }\r\n      p.info:before {\r\n        background: #0066CC;\r\n        border-radius: 16px;\r\n        color: #fff;\r\n        content: \"i\";\r\n        font: bold 16px/24px serif;\r\n        height: 24px;\r\n        left: 0px;\r\n        text-align: center;\r\n        top: 4px;\r\n        width: 24px;\r\n      }\r\n\r\n      @media (min-width: 768px) {\r\n        body {\r\n          margin: 6em;\r\n        }\r\n      }\r\n    \u003c/style\u003e\r\n  \u003c/head\u003e\r\n  \u003cbody\u003e\r\n    \u003cdiv\u003e\r\n      \u003ch1\u003eApplication is not available\u003c/h1\u003e\r\n      \u003cp\u003eThe application is currently not serving requests at this endpoint. It may not have been started or is still starting.\u003c/p\u003e\r\n\r\n      \u003cdiv class=\"alert alert-info\"\u003e\r\n        \u003cp class=\"info\"\u003e\r\n          Possible reasons you are seeing this page:\r\n        \u003c/p\u003e\r\n        \u003cul\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eThe host doesn't exist.\u003c/strong\u003e\r\n            Make sure the hostname was typed correctly and that a route matching this hostname exists.\r\n          \u003c/li\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eThe host exists, but doesn't have a matching path.\u003c/strong\u003e\r\n            Check if the URL path was typed correctly and that the route was created using the desired path.\r\n          \u003c/li\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eRoute and path matches, but all pods are down.\u003c/strong\u003e\r\n            Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.\r\n          \u003c/li\u003e\r\n        \u003c/ul\u003e\r\n      \u003c/div\u003e\r\n    \u003c/div\u003e\r\n  \u003c/body\u003e\r\n\u003c/html\u003e"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074027709368897536","occurred_at":"2026-07-06T07:08:34Z","signature_id":"beed6c8947077f0d05c5c39f37785327d2c5b2056bbf6cbf83cf5591985b568b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074027709368897536","occurred_at":"2026-07-06T07:08:34Z","signature_id":"beed6c8947077f0d05c5c39f37785327d2c5b2056bbf6cbf83cf5591985b568b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074027709368897536","occurred_at":"2026-07-06T07:08:34Z","signature_id":"beed6c8947077f0d05c5c39f37785327d2c5b2056bbf6cbf83cf5591985b568b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"9d76b8574a881c48ca814c73a5f78cfd9c34fbf0c2d12d09c9f9b6e856adedcf","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":7.142857142857143,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,0,1,1,0,0,0,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-29"],"prior_runs_affected":3,"prior_last_seen_at":"2026-07-02T23:00:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0027903a8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-qt596f, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-qt596f\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0027903a8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-qt596f, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-qt596f\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013f8420\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-5rpjlm, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-5rpjlm\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013f8420\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-5rpjlm, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-5rpjlm\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074756168701448192","occurred_at":"2026-07-08T07:23:12Z","signature_id":"50ee8b735b3c0656fcc2610dc92456af6e5718a5356cf4d286a308333584b8f4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074498265448976384","occurred_at":"2026-07-07T14:18:23Z","signature_id":"4b7652f80ed03439468763a73437788cecbc7ae7fcac59db1e39b07e9dba0231","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"c10c7cf0ceed874f5e112f080a8feea94b6e28417c30e3d6b1b02b8d9b239205","failure_pattern":"after 4 attempts: ResponseFailed: unexpected HTTP status: 504 Gateway Timeout","search_query":"after 4 attempts: ResponseFailed: unexpected HTTP status: 504 Gateway Timeout","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,2,0,0,0,0,0,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:677]: failed to get latest version for minor 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010be160\u003e: \n    after 4 attempts: ResponseFailed: unexpected HTTP status: 504 Gateway Timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:677]: failed to get latest version for minor 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010be160\u003e: \n    after 4 attempts: ResponseFailed: unexpected HTTP status: 504 Gateway Timeout\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:65]: failed to determine z-stream install version for minor 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c2e120\u003e: \n    after 4 attempts: ResponseFailed: unexpected HTTP status: 504 Gateway Timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:65]: failed to determine z-stream install version for minor 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c2e120\u003e: \n    after 4 attempts: ResponseFailed: unexpected HTTP status: 504 Gateway Timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074027709368897536","occurred_at":"2026-07-06T07:08:34Z","signature_id":"240be092796a2285c51166d430f1be1298e773f08d7afe7494091aee4ea3f5c9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074027709368897536","occurred_at":"2026-07-06T07:08:34Z","signature_id":"304149e61cf382876ff4c35985e7caf64e2a3016db01f3e2b55859234f8e9aeb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b64b4ef0c88d35afab6c6cd8765ff6a9da8a5a8ffc9a639da00c5e1f9e608e45","failure_pattern":"ERROR CODE: NotFound; detail message Resource '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/e2e-oidc-wi-test' was not found.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: NotFound","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:296]: failed to create federated identity credential for service account system:serviceaccount:e2e-oidc-wi:wi-test-sa\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0024efa70\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-m6pkth/providers/Microsoft.ManagedIdentity/userAssignedIdentities/e2e-oidc-wi-test/federatedIdentityCredentials/e2e-wi-fic\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NotFound\",\n        \"message\": \"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/oidc-wi-m6pkth/providers/Microsoft.ManagedIdentity/userAssignedIdentities/e2e-oidc-wi-test' was not found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:296]: failed to create federated identity credential for service account system:serviceaccount:e2e-oidc-wi:wi-test-sa\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0024efa70\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-m6pkth/providers/Microsoft.ManagedIdentity/userAssignedIdentities/e2e-oidc-wi-test/federatedIdentityCredentials/e2e-wi-fic\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NotFound\",\n        \"message\": \"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/oidc-wi-m6pkth/providers/Microsoft.ManagedIdentity/userAssignedIdentities/e2e-oidc-wi-test' was not found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2076124508074283008","occurred_at":"2026-07-12T02:00:29Z","signature_id":"697ed4f1fa31f906da6e81d1dba09423da734b4de942cfb34f304a40f53ca43b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"4d71f4c6773e43c69a9b52acf91cc073640396aeef73c6b89e32ba79c3e1f4a8","failure_pattern":"VerifyFIPSEnabled failed: failed to check FIPS mode on node \u003cnode\u003e: failed to create FIPS check pod: pod \"\u003cpod\u003e\" is forbidden: error looking up service account \u003cnamespace\u003e/default: serviceaccount \"default\" not found","search_query":"VerifyFIPSEnabled failed: failed to check FIPS mode on node x6i8r8c0s5b9y1e-np-1-xdg6k-5zg5s: failed to create FIPS check pod: pods \"fips-check-\" is forbidden: error looking up service account e2e-fips-test-qvwcn/default: serviceaccount \"default\" not found","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc00107bab8\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node x6i8r8c0s5b9y1e-np-1-xdg6k-5zg5s: failed to create FIPS check pod: pods \"fips-check-\" is forbidden: error looking up service account e2e-fips-test-qvwcn/default: serviceaccount \"default\" not found\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc00107bab8\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node x6i8r8c0s5b9y1e-np-1-xdg6k-5zg5s: failed to create FIPS check pod: pods \"fips-check-\" is forbidden: error looking up service account e2e-fips-test-qvwcn/default: serviceaccount \"default\" not found\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074756168701448192","occurred_at":"2026-07-08T07:23:12Z","signature_id":"9db72c858ff1e39333cf54ce0f2615279998fb8e9d07f0ff9fa4bd650308750b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"5fa14ac2191be7f8a9cc5a8dcea50ff5b05e11e3f74856ddcd433ed61cc44378","failure_pattern":"failed to get latest install version for candidate channel: after 4 attempts: query candidate graph for \u003cocp-channel\u003e returned 504 Gateway Timeout: upstream request timeout","search_query":"fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:179]: failed to get latest install version for candidate channel: after 4 attempts: query candidate graph for candidate-4.20 returned 504 Gateway Timeout: upstream request timeout","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:179]: failed to get latest install version for candidate channel: after 4 attempts: query candidate graph for candidate-4.20 returned 504 Gateway Timeout: upstream request timeout\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:179]: failed to get latest install version for candidate channel: after 4 attempts: query candidate graph for candidate-4.20 returned 504 Gateway Timeout: upstream request timeout"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074027709368897536","occurred_at":"2026-07-06T07:08:34Z","signature_id":"92dfb6ec8fb9f4373c5df7cc8444d703b51c65852512396798ee4fa8856ab9cb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"8876cd0c014eeffce1f260b0253b73f276e0c8de69705d1e4548e4b9faf30778","failure_pattern":"failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish getting creds: Get \"\u003curl\u003e\": stream error: stream ID 175; CANCEL; received from peer","search_query":"failed waiting for hcpCluster=\"cluster-zstream-4-22-dxshzj\" in resourcegroup=\"rg-zstream-upgrade-4-22-9npmw6\" to finish getting creds: Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b1e7606d-ac0b-4b22-9951-0d6139fef826?api-version=2024-06-10-preview\u0026t=639190961451825049\u0026c=MIIHkTCCBnmgAwIBAgIRAIphu0X3wTKY-ML3gzs6k8owDQYJKoZIhvcNAQELBQAwNjE0MDIGA1UEAxMrQ0NNRSBHMSBUTFMgUlNBIDIwNDggU0hBMjU2IDIwNDkgRVVTMiBDQSAwMTAeFw0yNjA0MDUxOTUxNDZaFw0yNjEwMDEwMTUxNDZaMEAxPjA8BgNVBAMTNWFzeW5jb3BlcmF0aW9uc2lnbmluZ2NlcnRpZmljYXRlLm1hbmFnZW1lbnQuYXp1cmUuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnR-Iz8Fgy9ddzo1kN9_c9HX8fqbMo-rawsyhsE9PSveyJv-L6NSkYs-1wxuKxocBLyNq8Jah1OkhbHL2yHbaT2kH5ZLNyn03w4Fn5lmjGi0KRYdPQwRIoK_PTIYI3maaXA7Kya68KMyhJgqlwqg9IsBK3Dq2D3vr1XP-PogUr3cssjfqfW_pVnR8bWGDNVCFKHbZbXMBWTNpAW91fHtf-4ojMqnePnvHVqONRoxeIQxsLtx3bQSh4Hw2QF_n1pMCUJ52n2c3_kjlULgAIesOTCdL9Pqul7Qby0zkLW0k9ppiEtPlfX0NNTyex0tR5-UE6EfjaiZZq9c_3Er-iDFcyQIDAQABo4IEjjCCBIowgZ0GA1UdIASBlTCBkjAMBgorBgEEAYI3ewEBMGYGCisGAQQBgjd7AgIwWDBWBggrBgEFBQcCAjBKHkgAMwAzAGUAMAAxADkAMgAxAC0ANABkADYANAAtADQAZgA4AGMALQBhADAANQA1AC0ANQBiAGQAYQBmAGYAZAA1AGUAMwAzAGQwDAYKKwYBBAGCN3sDAjAMBgorBgEEAYI3ewQCMAwGA1UdEwEB_wQCMAAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMA4GA1UdDwEB_wQEAwIFoDAdBgNVHQ4EFgQU3jhSRQ1zlQcx7waACuHPEhCucfswHwYDVR0jBBgwFoAU_Ow-26p8H4IeBbihBvlD5wKzCrkwggGuBgNVHR8EggGlMIIBoTBooGagZIZiaHR0cDovL3ByaW1hcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L2Vhc3R1czIvY3Jscy9jY21lZWFzdHVzMnBraS9jY21lZWFzdHVzMmljYTAxLzMvY3VycmVudC5jcmwwaqBooGaGZGh0dHA6Ly9zZWNvbmRhcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L2Vhc3R1czIvY3Jscy9jY21lZWFzdHVzMnBraS9jY21lZWFzdHVzMmljYTAxLzMvY3VycmVudC5jcmwwWaBXoFWGU2h0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS9lYXN0dXMyL2NybHMvY2NtZWVhc3R1czJwa2kvY2NtZWVhc3R1czJpY2EwMS8zL2N1cnJlbnQuY3JsMG6gbKBqhmhodHRwOi8vY2NtZWVhc3R1czJwa2kuZWFzdHVzMi5wa2kuY29yZS53aW5kb3dzLm5ldC9jZXJ0aWZpY2F0ZUF1dGhvcml0aWVzL2NjbWVlYXN0dXMyaWNhMDEvMy9jdXJyZW50LmNybDCCAbcGCCsGAQUFBwEBBIIBqTCCAaUwbAYIKwYBBQUHMAKGYGh0dHA6Ly9wcmltYXJ5LWNkbi5wa2kuY29yZS53aW5kb3dzLm5ldC9lYXN0dXMyL2NhY2VydHMvY2NtZWVhc3R1czJwa2kvY2NtZWVhc3R1czJpY2EwMS9jZXJ0LmNlcjBuBggrBgEFBQcwAoZiaHR0cDovL3NlY29uZGFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvZWFzdHVzMi9jYWNlcnRzL2NjbWVlYXN0dXMycGtpL2NjbWVlYXN0dXMyaWNhMDEvY2VydC5jZXIwXQYIKwYBBQUHMAKGUWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS9lYXN0dXMyL2NhY2VydHMvY2NtZWVhc3R1czJwa2kvY2NtZWVhc3R1czJpY2EwMS9jZXJ0LmNlcjBmBggrBgEFBQcwAoZaaHR0cDovL2NjbWVlYXN0dXMycGtpLmVhc3R1czIucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21lZWFzdHVzMmljYTAxMA0GCSqGSIb3DQEBCwUAA4IBAQAnTNmUYMgtA5djeWjFRnP8bzb2RSij05KzI6-zmrz8PwFhloqF_K-ppJyIpYnHi-6ahqLnr9NDS-inELxBS21PcqBnDx4YKiKbvb4iFBRzM1_p-Qs_Ohg8kxB7_s3yJJ42MBeISb04d8hwwG0Hzy6JDsvrcnavrUJxzl7SoGQI3cdVs3L3rOxBKrmixXdFJBIOLZeN8ZRU9cs7iMLzt3XqSfAQv2DpEbhLst8W5F29nJyo_XsgU4CHTh6VDbniACMzAYRvF-MXIf2gLnjEuX4FhnaYly_84KVjx8sr1f0nqFtfDL6X6yYAtrzSZb5lv2fBipHiLwLRt3loXadj5jPy\u0026s=b5ROMIPn-mzkFhZhx7NJy1eqc88KFf26H3WdYCh2E00wIcC9xmZXEROuJk7fudR8yGlPltARYfDbKOKpcSYkLA5A6gwCpT2pH4NsmupfNRPd-P8xe0J2m45AxWZR-WtcVBKv3VNxXAbyTaJSo84LXGM7ST5afJo5LNpKcayqTDk9axgkBPzAHyy-h9WqL7x--k2U13Ypz5eZmoxTZai4_T0miL8G0wDuaCV8LHFDCSWgs-36AXlsHqQxcxKfPIivGQGUbs1cJmW8gkDwi3Ros2mPJz455jFhvHTweA_otcCI4aTcEPscwe91BPfUjtVgAnLzQJnm6_hBdI4X7qPt-g\u0026h=H7Zn71yX1kKnQgEZVybYJDSWtRW2B5SVmIDaVUuNXsQ\": stream error: stream ID 175; CANCEL; received from peer","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:119]: failed to get admin REST config for cluster \"cluster-zstream-4-22-dxshzj\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005104c0\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-22-dxshzj\" in resourcegroup=\"rg-zstream-upgrade-4-22-9npmw6\" to finish getting creds: Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b1e7606d-ac0b-4b22-9951-0d6139fef826?api-version=2024-06-10-preview\u0026t=639190961451825049\u0026c=MIIHkTCCBnmgAwIBAgIRAIphu0X3wTKY-ML3gzs6k8owDQYJKoZIhvcNAQELBQAwNjE0MDIGA1UEAxMrQ0NNRSBHMSBUTFMgUlNBIDIwNDggU0hBMjU2IDIwNDkgRVVTMiBDQSAwMTAeFw0yNjA0MDUxOTUxNDZaFw0yNjEwMDEwMTUxNDZaMEAxPjA8BgNVBAMTNWFzeW5jb3BlcmF0aW9uc2lnbmluZ2NlcnRpZmljYXRlLm1hbmFnZW1lbnQuYXp1cmUuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnR-Iz8Fgy9ddzo1kN9_c9HX8fqbMo-rawsyhsE9PSveyJv-L6NSkYs-1wxuKxocBLyNq8Jah1OkhbHL2yHbaT2kH5ZLNyn03w4Fn5lmjGi0KRYdPQwRIoK_PTIYI3maaXA7Kya68KMyhJgqlwqg9IsBK3Dq2D3vr1XP-PogUr3cssjfqfW_pVnR8bWGDNVCFKHbZbXMBWTNpAW91fHtf-4ojMqnePnvHVqONRoxeIQxsLtx3bQSh4Hw2QF_n1pMCUJ52n2c3_kjlULgAIesOTCdL9Pqul7Qby0zkLW0k9ppiEtPlfX0NNTyex0tR5-UE6EfjaiZZq9c_3Er-iDFcyQIDAQABo4IEjjCCBIowgZ0GA1UdIASBlTCBkjAMBgorBgEEAYI3ewEBMGYGCisGAQQBgjd7AgIwWDBWBggrBgEFBQcCAjBKHkgAMwAzAGUAMAAxADkAMgAxAC0ANABkADYANAAtADQAZgA4AGMALQBhADAANQA1AC0ANQBiAGQAYQBmAGYAZAA1AGUAMwAzAGQwDAYKKwYBBAGCN3sDAjAMBgorBgEEAYI3ewQCMAwGA1UdEwEB_wQCMAAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMA4GA1UdDwEB_wQEAwIFoDAdBgNVHQ4EFgQU3jhSRQ1zlQcx7waACuHPEhCucfswHwYDVR0jBBgwFoAU_Ow-26p8H4IeBbihBvlD5wKzCrkwggGuBgNVHR8EggGlMIIBoTBooGagZIZiaHR0cDovL3ByaW1hcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L2Vhc3R1czIvY3Jscy9jY21lZWFzdHVzMnBraS9jY21lZWFzdHVzMmljYTAxLzMvY3VycmVudC5jcmwwaqBooGaGZGh0dHA6Ly9zZWNvbmRhcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L2Vhc3R1czIvY3Jscy9jY21lZWFzdHVzMnBraS9jY21lZWFzdHVzMmljYTAxLzMvY3VycmVudC5jcmwwWaBXoFWGU2h0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS9lYXN0dXMyL2NybHMvY2NtZWVhc3R1czJwa2kvY2NtZWVhc3R1czJpY2EwMS8zL2N1cnJlbnQuY3JsMG6gbKBqhmhodHRwOi8vY2NtZWVhc3R1czJwa2kuZWFzdHVzMi5wa2kuY29yZS53aW5kb3dzLm5ldC9jZXJ0aWZpY2F0ZUF1dGhvcml0aWVzL2NjbWVlYXN0dXMyaWNhMDEvMy9jdXJyZW50LmNybDCCAbcGCCsGAQUFBwEBBIIBqTCCAaUwbAYIKwYBBQUHMAKGYGh0dHA6Ly9wcmltYXJ5LWNkbi5wa2kuY29yZS53aW5kb3dzLm5ldC9lYXN0dXMyL2NhY2VydHMvY2NtZWVhc3R1czJwa2kvY2NtZWVhc3R1czJpY2EwMS9jZXJ0LmNlcjBuBggrBgEFBQcwAoZiaHR0cDovL3NlY29uZGFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvZWFzdHVzMi9jYWNlcnRzL2NjbWVlYXN0dXMycGtpL2NjbWVlYXN0dXMyaWNhMDEvY2VydC5jZXIwXQYIKwYBBQUHMAKGUWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS9lYXN0dXMyL2NhY2VydHMvY2NtZWVhc3R1czJwa2kvY2NtZWVhc3R1czJpY2EwMS9jZXJ0LmNlcjBmBggrBgEFBQcwAoZaaHR0cDovL2NjbWVlYXN0dXMycGtpLmVhc3R1czIucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21lZWFzdHVzMmljYTAxMA0GCSqGSIb3DQEBCwUAA4IBAQAnTNmUYMgtA5djeWjFRnP8bzb2RSij05KzI6-zmrz8PwFhloqF_K-ppJyIpYnHi-6ahqLnr9NDS-inELxBS21PcqBnDx4YKiKbvb4iFBRzM1_p-Qs_Ohg8kxB7_s3yJJ42MBeISb04d8hwwG0Hzy6JDsvrcnavrUJxzl7SoGQI3cdVs3L3rOxBKrmixXdFJBIOLZeN8ZRU9cs7iMLzt3XqSfAQv2DpEbhLst8W5F29nJyo_XsgU4CHTh6VDbniACMzAYRvF-MXIf2gLnjEuX4FhnaYly_84KVjx8sr1f0nqFtfDL6X6yYAtrzSZb5lv2fBipHiLwLRt3loXadj5jPy\u0026s=b5ROMIPn-mzkFhZhx7NJy1eqc88KFf26H3WdYCh2E00wIcC9xmZXEROuJk7fudR8yGlPltARYfDbKOKpcSYkLA5A6gwCpT2pH4NsmupfNRPd-P8xe0J2m45AxWZR-WtcVBKv3VNxXAbyTaJSo84LXGM7ST5afJo5LNpKcayqTDk9axgkBPzAHyy-h9WqL7x--k2U13Ypz5eZmoxTZai4_T0miL8G0wDuaCV8LHFDCSWgs-36AXlsHqQxcxKfPIivGQGUbs1cJmW8gkDwi3Ros2mPJz455jFhvHTweA_otcCI4aTcEPscwe91BPfUjtVgAnLzQJnm6_hBdI4X7qPt-g\u0026h=H7Zn71yX1kKnQgEZVybYJDSWtRW2B5SVmIDaVUuNXsQ\": stream error: stream ID 175; CANCEL; received from peer\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:119]: failed to get admin REST config for cluster \"cluster-zstream-4-22-dxshzj\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005104c0\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-22-dxshzj\" in resourcegroup=\"rg-zstream-upgrade-4-22-9npmw6\" to finish getting creds: Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b1e7606d-ac0b-4b22-9951-0d6139fef826?api-version=2024-06-10-preview\u0026t=639190961451825049\u0026c=MIIHkTCCBnmgAwIBAgIRAIphu0X3wTKY-ML3gzs6k8owDQYJKoZIhvcNAQELBQAwNjE0MDIGA1UEAxMrQ0NNRSBHMSBUTFMgUlNBIDIwNDggU0hBMjU2IDIwNDkgRVVTMiBDQSAwMTAeFw0yNjA0MDUxOTUxNDZaFw0yNjEwMDEwMTUxNDZaMEAxPjA8BgNVBAMTNWFzeW5jb3BlcmF0aW9uc2lnbmluZ2NlcnRpZmljYXRlLm1hbmFnZW1lbnQuYXp1cmUuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnR-Iz8Fgy9ddzo1kN9_c9HX8fqbMo-rawsyhsE9PSveyJv-L6NSkYs-1wxuKxocBLyNq8Jah1OkhbHL2yHbaT2kH5ZLNyn03w4Fn5lmjGi0KRYdPQwRIoK_PTIYI3maaXA7Kya68KMyhJgqlwqg9IsBK3Dq2D3vr1XP-PogUr3cssjfqfW_pVnR8bWGDNVCFKHbZbXMBWTNpAW91fHtf-4ojMqnePnvHVqONRoxeIQxsLtx3bQSh4Hw2QF_n1pMCUJ52n2c3_kjlULgAIesOTCdL9Pqul7Qby0zkLW0k9ppiEtPlfX0NNTyex0tR5-UE6EfjaiZZq9c_3Er-iDFcyQIDAQABo4IEjjCCBIowgZ0GA1UdIASBlTCBkjAMBgorBgEEAYI3ewEBMGYGCisGAQQBgjd7AgIwWDBWBggrBgEFBQcCAjBKHkgAMwAzAGUAMAAxADkAMgAxAC0ANABkADYANAAtADQAZgA4AGMALQBhADAANQA1AC0ANQBiAGQAYQBmAGYAZAA1AGUAMwAzAGQwDAYKKwYBBAGCN3sDAjAMBgorBgEEAYI3ewQCMAwGA1UdEwEB_wQCMAAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMA4GA1UdDwEB_wQEAwIFoDAdBgNVHQ4EFgQU3jhSRQ1zlQcx7waACuHPEhCucfswHwYDVR0jBBgwFoAU_Ow-26p8H4IeBbihBvlD5wKzCrkwggGuBgNVHR8EggGlMIIBoTBooGagZIZiaHR0cDovL3ByaW1hcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L2Vhc3R1czIvY3Jscy9jY21lZWFzdHVzMnBraS9jY21lZWFzdHVzMmljYTAxLzMvY3VycmVudC5jcmwwaqBooGaGZGh0dHA6Ly9zZWNvbmRhcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L2Vhc3R1czIvY3Jscy9jY21lZWFzdHVzMnBraS9jY21lZWFzdHVzMmljYTAxLzMvY3VycmVudC5jcmwwWaBXoFWGU2h0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS9lYXN0dXMyL2NybHMvY2NtZWVhc3R1czJwa2kvY2NtZWVhc3R1czJpY2EwMS8zL2N1cnJlbnQuY3JsMG6gbKBqhmhodHRwOi8vY2NtZWVhc3R1czJwa2kuZWFzdHVzMi5wa2kuY29yZS53aW5kb3dzLm5ldC9jZXJ0aWZpY2F0ZUF1dGhvcml0aWVzL2NjbWVlYXN0dXMyaWNhMDEvMy9jdXJyZW50LmNybDCCAbcGCCsGAQUFBwEBBIIBqTCCAaUwbAYIKwYBBQUHMAKGYGh0dHA6Ly9wcmltYXJ5LWNkbi5wa2kuY29yZS53aW5kb3dzLm5ldC9lYXN0dXMyL2NhY2VydHMvY2NtZWVhc3R1czJwa2kvY2NtZWVhc3R1czJpY2EwMS9jZXJ0LmNlcjBuBggrBgEFBQcwAoZiaHR0cDovL3NlY29uZGFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvZWFzdHVzMi9jYWNlcnRzL2NjbWVlYXN0dXMycGtpL2NjbWVlYXN0dXMyaWNhMDEvY2VydC5jZXIwXQYIKwYBBQUHMAKGUWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS9lYXN0dXMyL2NhY2VydHMvY2NtZWVhc3R1czJwa2kvY2NtZWVhc3R1czJpY2EwMS9jZXJ0LmNlcjBmBggrBgEFBQcwAoZaaHR0cDovL2NjbWVlYXN0dXMycGtpLmVhc3R1czIucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21lZWFzdHVzMmljYTAxMA0GCSqGSIb3DQEBCwUAA4IBAQAnTNmUYMgtA5djeWjFRnP8bzb2RSij05KzI6-zmrz8PwFhloqF_K-ppJyIpYnHi-6ahqLnr9NDS-inELxBS21PcqBnDx4YKiKbvb4iFBRzM1_p-Qs_Ohg8kxB7_s3yJJ42MBeISb04d8hwwG0Hzy6JDsvrcnavrUJxzl7SoGQI3cdVs3L3rOxBKrmixXdFJBIOLZeN8ZRU9cs7iMLzt3XqSfAQv2DpEbhLst8W5F29nJyo_XsgU4CHTh6VDbniACMzAYRvF-MXIf2gLnjEuX4FhnaYly_84KVjx8sr1f0nqFtfDL6X6yYAtrzSZb5lv2fBipHiLwLRt3loXadj5jPy\u0026s=b5ROMIPn-mzkFhZhx7NJy1eqc88KFf26H3WdYCh2E00wIcC9xmZXEROuJk7fudR8yGlPltARYfDbKOKpcSYkLA5A6gwCpT2pH4NsmupfNRPd-P8xe0J2m45AxWZR-WtcVBKv3VNxXAbyTaJSo84LXGM7ST5afJo5LNpKcayqTDk9axgkBPzAHyy-h9WqL7x--k2U13Ypz5eZmoxTZai4_T0miL8G0wDuaCV8LHFDCSWgs-36AXlsHqQxcxKfPIivGQGUbs1cJmW8gkDwi3Ros2mPJz455jFhvHTweA_otcCI4aTcEPscwe91BPfUjtVgAnLzQJnm6_hBdI4X7qPt-g\u0026h=H7Zn71yX1kKnQgEZVybYJDSWtRW2B5SVmIDaVUuNXsQ\": stream error: stream ID 175; CANCEL; received from peer\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074756168701448192","occurred_at":"2026-07-08T07:23:12Z","signature_id":"b32739bec44c69eaf4d0d26c444c9bb7dd664f084be28c07e0d7087537e00274","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"bc6feba968b96707a741a4a262cde01914a092f8c9a03969ae236fd2c5fbffe1","failure_pattern":"query graph for \u003cocp-channel\u003e: Get \"\u003curl\u003e\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","search_query":"query graph for candidate-4.20: Get \"https://api.openshift.com/api/upgrades_info/v1/graph?channel=candidate-4.20\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:257]: failed to get version pair without upgrade edge for minor 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00110cee0\u003e: \n    query graph for candidate-4.20: Get \"https://api.openshift.com/api/upgrades_info/v1/graph?channel=candidate-4.20\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:257]: failed to get version pair without upgrade edge for minor 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00110cee0\u003e: \n    query graph for candidate-4.20: Get \"https://api.openshift.com/api/upgrades_info/v1/graph?channel=candidate-4.20\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2074027709368897536","occurred_at":"2026-07-06T07:08:34Z","signature_id":"2d6ef40a50489d45b8c254165d438f724e56bde95bbfe36ff1392b7ae6593a79","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}