{"meta":{"start_date":"2026-07-10","end_date":"2026-07-10","start_at":"2026-07-10T00:00:00Z","end_at":"2026-07-11T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T07:01:37Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":3,"failed_runs":2,"raw_occurrences":46,"matched_occurrences":46,"runs_affected":2},"rows":[{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-22-fhwx4w, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-fhwx4w in resource group rg-candidate-4-22-vdpqsc, error: failed waiting for cluster=\"cluster-candidate-4-22-fhwx4w\" in resourcegroup=\"rg-candidate-4-22-vdpqsc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-fhwx4w in resource group rg-candidate-4-22-vdpqsc, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":30,"runs_affected":2,"failed_runs":2,"run_impact_percent":66.66666666666667,"anchor_occurrences":30,"after_last_push_count":30,"trend_counts":[0,0,0,0,0,0,30],"trend_range":"2026-07-04..2026-07-10 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-08","2026-06-15","2026-06-22","2026-06-29"],"prior_runs_affected":67,"prior_last_seen_at":"2026-07-04T02:00:13Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-4-23-48jxl4/cluster-nightly-4-23-52xj5n should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013bc570\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-52xj5n, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-52xj5n in resource group rg-nightly-4-23-48jxl4, error: failed waiting for cluster=\"cluster-nightly-4-23-52xj5n\" in resourcegroup=\"rg-nightly-4-23-48jxl4\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-52xj5n in resource group rg-nightly-4-23-48jxl4, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-4-23-48jxl4/cluster-nightly-4-23-52xj5n should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013bc570\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-52xj5n, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-52xj5n in resource group rg-nightly-4-23-48jxl4, error: failed waiting for cluster=\"cluster-nightly-4-23-52xj5n\" in resourcegroup=\"rg-nightly-4-23-48jxl4\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-52xj5n in resource group rg-nightly-4-23-48jxl4, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:86]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0005be990\u003e: \n    failed to create HCP cluster ea-listc4gkh5, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listc4gkh5 in resource group ea-list-br98kr, error: failed waiting for cluster=\"ea-listc4gkh5\" in resourcegroup=\"ea-list-br98kr\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listc4gkh5 in resource group ea-list-br98kr, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:86]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0005be990\u003e: \n    failed to create HCP cluster ea-listc4gkh5, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listc4gkh5 in resource group ea-list-br98kr, error: failed waiting for cluster=\"ea-listc4gkh5\" in resourcegroup=\"ea-list-br98kr\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listc4gkh5 in resource group ea-list-br98kr, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00135e570\u003e: \n    failed to create HCP cluster cluster-np-labels-taints, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-lzkfm5, error: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-lzkfm5\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-lzkfm5, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00135e570\u003e: \n    failed to create HCP cluster cluster-np-labels-taints, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-lzkfm5, error: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-lzkfm5\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-lzkfm5, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2075716817757671424","occurred_at":"2026-07-10T23:00:28Z","signature_id":"17d4ef6a848ccbd9a1c3983904d707817e3a3eedce0da626c102274b96c1db15","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"0003cabe51c1c01edcb2207e546b38cad1eae96c99e9cecf2d3c6f64b39be3e3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"094d5f5e30b5ea4bc3960153778f689f00351509fed8c4d99d6d7fb1ec9aed55","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"108edf565c7eca2464b51bdfbc106c93d811e429e473079818a5537222fa5e50","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"23bf1d14de43c5f95058a5b8512d2cfd9a1e696f5d43b77ed656b42a8f36074b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"25adc7e64dc1a3956a205646cabb071643e30520bbee959dc1ddba8534183c9d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"27ef1864ed49017a7e216dbb60d6b5ea8054a563b1a670789c285ba87e3c7297","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"297cfe1effc1bd34d70033dd09ac971dd452fc5845964218cd679f4317512385","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"37754912d4b084426e70c9a109d25bc03a70fab94fd38dc2ad8209fbd140d07d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"63a4f0336eb18341eefc1311f6c2cda4f917468e46c216049b76e146b4f628eb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"659af3978e8b63e1b66c619018abba7a5891eeede1fc839f579cba93cb496284","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"6d3c114015f23efa33a47d41dc04e8b4dc6a37c2de0dbc3d88a60d9602e1d297","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"7070e25b198c3dca364d18b9267309f841f52e3f6fc1815b211ccc0d000ff17a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"71d72283fa52a0c6b3ef69aa6389b5128c152564a3818742b154635f6161b701","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"79bf39ab5f7656b87e5f04884f042cbe7ddb1b4cf75b0c5b22c2a032d2883439","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"82a7d8496376bc37f8f7e05b26f76c937005216b454b654b70d4833a71b14548","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"97283cff250d9b63f671f55c678d424bc068adb9a64d9850cc29b6af2dc1f40e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"9a572401b3ff166dbcf39dc856299ae13d6ec7d955f0f77678ad3f1ffb9939cd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"a13d92c755b8f153d4c9924b655977660ce8d9ad3c8832d886c74d3763c61a07","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"a941afb00367f2d9cf5d057af6330b06d2e8da2f1f0eaac31fbec1c9bdce7fcb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"b768211abdca0d09d14f4f2a0125fce6e4199cf7faea8c048fa3652cb3b70de4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"bf6a575952021000e0bf74b46fb7a94fef94cae021c743a060425653be86e6a5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"d7e2bb0b213890c4ece1b37b7e025da6bc5690d9f9c38e40d30e5677ed395ee3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"dce4bd90df80eaeebeec2fd0c8f8e80ce6374b5efd61d38a15b5c398f9809e8d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"de14097e126d959dcae731a9f5484116a319647b09e278ea751efda382648b38","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"e430d5414e4a522a155d84bafff228baf8477cb11a46be2a3c89ef3521f8fc2a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"e5de09db273887caf7fcf0d2a52d3e1801ad10d951894bb9aaa19ef4f5b5e140","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"ef684e3e4bc592f98641ceff93349beb5e29367e04c57f0fc902dbdc6d8617a6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"efd93abd2dfd31d79e2528d9276753535ec8125a323e05799fc422d14c41b6e8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"f9a4a62075686e33ee6e8400132df86ceb04e7e97b495667ce94c4f21d77e85c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"admin-credential-lifecycle-test-glprsp\", caused by: failed waiting for hcpCluster=\"admin-cred-lifecycle-n7dbhc\" in resourcegroup=\"admin-credential-lifecycle-test-glprsp\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-n7dbhc in resource group admin-credential-lifecycle-test-glprsp, error: context deadline exceeded, error: failed waiting for hcpCluster=\"admin-cred-lifecycle-n7dbhc\" in resourcegroup=\"admin-credential-lifecycle-test-glprsp\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-n7dbhc in resource group admin-credential-lifecycle-test-glprsp, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":6,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":6,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,0,6],"trend_range":"2026-07-04..2026-07-10 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc002730108\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-skip-minor-j5x5g5-c9l8qq\", caused by: failed waiting for hcpCluster=\"np-skip-minor-j5x5g5\" in resourcegroup=\"rg-np-skip-minor-j5x5g5-c9l8qq\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-j5x5g5 in resource group rg-np-skip-minor-j5x5g5-c9l8qq, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-skip-minor-j5x5g5\" in resourcegroup=\"rg-np-skip-minor-j5x5g5-c9l8qq\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-j5x5g5 in resource group rg-np-skip-minor-j5x5g5-c9l8qq, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc002730108\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-skip-minor-j5x5g5-c9l8qq\", caused by: failed waiting for hcpCluster=\"np-skip-minor-j5x5g5\" in resourcegroup=\"rg-np-skip-minor-j5x5g5-c9l8qq\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-j5x5g5 in resource group rg-np-skip-minor-j5x5g5-c9l8qq, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-skip-minor-j5x5g5\" in resourcegroup=\"rg-np-skip-minor-j5x5g5-c9l8qq\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-j5x5g5 in resource group rg-np-skip-minor-j5x5g5-c9l8qq, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010aa120\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"admin-credential-lifecycle-test-glprsp\", caused by: failed waiting for hcpCluster=\"admin-cred-lifecycle-n7dbhc\" in resourcegroup=\"admin-credential-lifecycle-test-glprsp\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-n7dbhc in resource group admin-credential-lifecycle-test-glprsp, error: context deadline exceeded, error: failed waiting for hcpCluster=\"admin-cred-lifecycle-n7dbhc\" in resourcegroup=\"admin-credential-lifecycle-test-glprsp\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-n7dbhc in resource group admin-credential-lifecycle-test-glprsp, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010aa120\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"admin-credential-lifecycle-test-glprsp\", caused by: failed waiting for hcpCluster=\"admin-cred-lifecycle-n7dbhc\" in resourcegroup=\"admin-credential-lifecycle-test-glprsp\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-n7dbhc in resource group admin-credential-lifecycle-test-glprsp, error: context deadline exceeded, error: failed waiting for hcpCluster=\"admin-cred-lifecycle-n7dbhc\" in resourcegroup=\"admin-credential-lifecycle-test-glprsp\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-n7dbhc in resource group admin-credential-lifecycle-test-glprsp, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc001f842d0\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-wkj8vb\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-wkj8vb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-wkj8vb, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-wkj8vb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-wkj8vb, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc001f842d0\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-wkj8vb\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-wkj8vb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-wkj8vb, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-wkj8vb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-wkj8vb, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2075716817757671424","occurred_at":"2026-07-10T23:00:28Z","signature_id":"207329864642edc3f0c3d96e750c0643f470b9f6d35abc76bb44275dcecb7618","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2075716817757671424","occurred_at":"2026-07-10T23:00:28Z","signature_id":"3d9339b76e37c4f798a9c837c2bedc53796f83f95f3d29d5825aa3ad5a44a388","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2075716817757671424","occurred_at":"2026-07-10T23:00:28Z","signature_id":"57087a845372e6faa22a805ca4567c088ce184b3f64eda5f3774d6aa226913be","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2075716817757671424","occurred_at":"2026-07-10T23:00:28Z","signature_id":"91b2d6e4c499e4d9e6bc34c0a15f3523eaba9a94fb7c5fb87a4697d191b524c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2075716817757671424","occurred_at":"2026-07-10T23:00:28Z","signature_id":"eba1a221dd0bd497896302805295877c144206e4d24929d6412685279740852c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2075716817757671424","occurred_at":"2026-07-10T23:00:28Z","signature_id":"ec6a246836da64ca449ea7b32a8bf2bbdb462b1cacf500755fc659230f0f34e6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f27b30bef0ebb941cd79b083d3eb5ff3005b4b85f92582e4349d8748b1eaeb94","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-hbdz7m\" to finish deleting: context canceled","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-07-04..2026-07-10 UTC","prior_windows_present":2,"prior_window_starts":["2026-06-15","2026-06-22"],"prior_runs_affected":7,"prior_last_seen_at":"2026-06-23T23:00:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00169e090\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-hbdz7m\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00169e090\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-hbdz7m\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000a20330\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-8s4rn7\" in resourcegroup=\"rg-zstream-upgrade-4-20-tp4pvb\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000a20330\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-8s4rn7\" in resourcegroup=\"rg-zstream-upgrade-4-20-tp4pvb\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000d4b050\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-rvc9mn\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000d4b050\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-rvc9mn\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"3e5a6c977e959ca69eb1e1f2872b2dc7084d03a1803f8929cfba65b48edbf01c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"58c5fbb1501b065dbc1c9b23e11576aa0fcb1713cea279e948cf9be9d20e61ca","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"eee44a43fcd7a9a471f535f17ae4c45dfbaac3b8bb7f3fc7de2d77974828bba8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"fd64f8d7a67988e152ec5e7c782f9bd72cb8a1a4419ea2fa530aafd1c26d041a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-8k9j5p, error: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-8k9j5p\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-07-04..2026-07-10 UTC","prior_windows_present":3,"prior_window_starts":["2026-06-08","2026-06-15","2026-06-22"],"prior_runs_affected":34,"prior_last_seen_at":"2026-06-23T01:55:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-s2txmj\" for cluster \"cluster-candidate-4-21-s2txmj\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001b15ec0\u003e: \n    failed to create NodePool np-s2txmj, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-s2txmj in resource group rg-candidate-4-21-hlz6pz, error: failed waiting for nodepool=\"np-s2txmj\" for cluster \"cluster-candidate-4-21-s2txmj\" in resourcegroup=\"rg-candidate-4-21-hlz6pz\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-s2txmj\" for cluster \"cluster-candidate-4-21-s2txmj\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001b15ec0\u003e: \n    failed to create NodePool np-s2txmj, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-s2txmj in resource group rg-candidate-4-21-hlz6pz, error: failed waiting for nodepool=\"np-s2txmj\" for cluster \"cluster-candidate-4-21-s2txmj\" in resourcegroup=\"rg-candidate-4-21-hlz6pz\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-sfgvlw\" for cluster \"cluster-candidate-5-0-sfgvlw\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00030d050\u003e: \n    failed to create NodePool np-sfgvlw, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-sfgvlw in resource group rg-candidate-5-0-dl7nwv, error: failed waiting for nodepool=\"np-sfgvlw\" for cluster \"cluster-candidate-5-0-sfgvlw\" in resourcegroup=\"rg-candidate-5-0-dl7nwv\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-sfgvlw\" for cluster \"cluster-candidate-5-0-sfgvlw\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00030d050\u003e: \n    failed to create NodePool np-sfgvlw, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-sfgvlw in resource group rg-candidate-5-0-dl7nwv, error: failed waiting for nodepool=\"np-sfgvlw\" for cluster \"cluster-candidate-5-0-sfgvlw\" in resourcegroup=\"rg-candidate-5-0-dl7nwv\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:259]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002488cc0\u003e: \n    failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-8k9j5p, error: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-8k9j5p\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:259]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002488cc0\u003e: \n    failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-8k9j5p, error: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-8k9j5p\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"0b5027f5de34123446ebe5f45a2f704fb24995ace2cc881f916a0947a5ed0357","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"1142b94be10c149891bc7663ac8d0a8ff146939fd5db89bf7f3b17aa55bd07b8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"52aa2ed9632a5196773fdccfac3ac521f9c3090d5de6ac41965e72a91450bdf5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"cluster-zstream-4-22-9mnhf7\" in resourcegroup=\"rg-zstream-upgrade-4-22-6vc2xq\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-22-9mnhf7 in resource group rg-zstream-upgrade-4-22-6vc2xq, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":66.66666666666667,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-07-04..2026-07-10 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-15"],"prior_runs_affected":4,"prior_last_seen_at":"2026-06-17T10:52:58Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:119]: failed to get admin REST config for HCP cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001cfeba0\u003e: \n    failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-pqvwsl\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster fips-enabled-cluster in resource group fips-enabled-pqvwsl, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:119]: failed to get admin REST config for HCP cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001cfeba0\u003e: \n    failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-pqvwsl\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster fips-enabled-cluster in resource group fips-enabled-pqvwsl, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:119]: failed to get admin REST config for cluster \"cluster-zstream-4-22-9mnhf7\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0005f6cf0\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-22-9mnhf7\" in resourcegroup=\"rg-zstream-upgrade-4-22-6vc2xq\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-22-9mnhf7 in resource group rg-zstream-upgrade-4-22-6vc2xq, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:119]: failed to get admin REST config for cluster \"cluster-zstream-4-22-9mnhf7\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0005f6cf0\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-22-9mnhf7\" in resourcegroup=\"rg-zstream-upgrade-4-22-6vc2xq\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-22-9mnhf7 in resource group rg-zstream-upgrade-4-22-6vc2xq, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2075716817757671424","occurred_at":"2026-07-10T23:00:28Z","signature_id":"f2a000470a0574bdb22aea8c479c3d131c656065568d206765f6f170da921cde","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"85d3c395cf9942afe6177cadcf6940fd336434719405e671310968feb2408307","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dcad60eb8db626db10bfb0812a18e7c392049987a81c4fcbcdc5140e0032dc41","failure_pattern":"Cluster should become ready within '20.000000' minutes","search_query":"Cluster should become ready within '20.000000' minutes","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":7,"trend_counts":[0,0,0,0,1,0,1],"trend_range":"2026-07-04..2026-07-10 UTC","prior_windows_present":3,"prior_window_starts":["2026-06-08","2026-06-15","2026-06-22"],"prior_runs_affected":11,"prior_last_seen_at":"2026-06-23T05:14:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 1200.004s.\nCluster should become ready within '20.000000' minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 1200.004s.\nCluster should become ready within '20.000000' minutes\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"7653ffd4e02735b234dd652c0dd32332c9566363ef7806113eb84037d3ada7a9","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}