{"meta":{"start_date":"2026-07-06","end_date":"2026-07-12","start_at":"2026-07-06T00:00:00Z","end_at":"2026-07-13T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T04:34:29Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":28,"failed_runs":20,"raw_occurrences":178,"matched_occurrences":177,"runs_affected":19},"rows":[{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":123,"runs_affected":8,"failed_runs":8,"run_impact_percent":28.571428571428573,"anchor_occurrences":123,"after_last_push_count":123,"trend_counts":[0,0,0,45,2,30,46,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-08","2026-06-15","2026-06-22","2026-06-29"],"prior_runs_affected":69,"prior_last_seen_at":"2026-07-04T02:00:13Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-19-v2vs79\" with version 4.19.35 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ce48a0\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-v2vs79, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-v2vs79 in resource group rg-zstream-upgrade-4-19-j968pf, error: failed waiting for cluster=\"cluster-zstream-4-19-v2vs79\" in resourcegroup=\"rg-zstream-upgrade-4-19-j968pf\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-v2vs79 in resource group rg-zstream-upgrade-4-19-j968pf, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-19-v2vs79\" with version 4.19.35 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ce48a0\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-v2vs79, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-v2vs79 in resource group rg-zstream-upgrade-4-19-j968pf, error: failed waiting for cluster=\"cluster-zstream-4-19-v2vs79\" in resourcegroup=\"rg-zstream-upgrade-4-19-j968pf\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-v2vs79 in resource group rg-zstream-upgrade-4-19-j968pf, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:159]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014e2420\u003e: \n    failed to create HCP cluster patch-tags-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-kn28ct, error: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-kn28ct\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-kn28ct, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:159]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014e2420\u003e: \n    failed to create HCP cluster patch-tags-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-kn28ct, error: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-kn28ct\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-kn28ct, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-20-ndvw6g\" with version 4.20.27 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009a07e0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-ndvw6g, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-ndvw6g in resource group rg-zstream-upgrade-4-20-9crtsh, error: failed waiting for cluster=\"cluster-zstream-4-20-ndvw6g\" in resourcegroup=\"rg-zstream-upgrade-4-20-9crtsh\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-ndvw6g in resource group rg-zstream-upgrade-4-20-9crtsh, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-20-ndvw6g\" with version 4.20.27 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009a07e0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-ndvw6g, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-ndvw6g in resource group rg-zstream-upgrade-4-20-9crtsh, error: failed waiting for cluster=\"cluster-zstream-4-20-ndvw6g\" in resourcegroup=\"rg-zstream-upgrade-4-20-9crtsh\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-ndvw6g in resource group rg-zstream-upgrade-4-20-9crtsh, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2076079208907411456","occurred_at":"2026-07-11T23:00:29Z","signature_id":"295d0ae373f787db7cc03ae271bcd03f44425d98e5b3f6ee496841e90aa0fe28","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2076079208907411456","occurred_at":"2026-07-11T23:00:29Z","signature_id":"4203a2f72ed0c06e15e65d91d1538c714beb38fe96064eb4470ebdbb4bf0f7cd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2076079208907411456","occurred_at":"2026-07-11T23:00:29Z","signature_id":"57f3aae6edcb103ecb4092a0542b513cd2011702b1b90bd88d3b577f5cfb1110","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2076079208907411456","occurred_at":"2026-07-11T23:00:29Z","signature_id":"5d084c3f3c606aeca0e2c7561717fb59ee835f9af0626324fed12dd974b1ffe7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2076079208907411456","occurred_at":"2026-07-11T23:00:29Z","signature_id":"fe83eb2e1faf35ccc0d8b9bef9c33b9bab9aefed6291e2bd9c92efc646f04523","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"08318022d91a1ebe9569520821c87125282eaacc8851c8ca08ea8834f0bcb0b3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"116964d308f699f4a9b204561316f4987e8e395c1760bea603cc3772dc7cad50","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"119c4e0b6d0cd0db96e1003b9c5342c2a3f068f6ecad46837ad2686c0da7f099","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"14ef28aa9650b0f3a94e1b316a686e927de159bb8be9dc207bf5914e64f45ff4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"1f4f602d17a576b45b6b754dcbd04266c44363fc4bb256b43a4a538de5b874b0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"23f8aedf2ed982edb9355aec6a186763bec349d317da50d2f0e361d34bdab7ec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"26f70fbec1d9c114069fdef5d2735b49df750d6d95d6493d17acd9a8899eee72","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"2987c8f4c655ed52d0adcc0f710f0da9b23f651ce5736eda54a5fac6eff6af56","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"3020355086f3ca721d4c918bc7526f2e0d4aa02130d0508ea39bea2e5204e8a2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"3096a8feded2ff270a7f17e3169e696f90f7495aacd5cb195ae4f78921923e42","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"3bc6fe6265f3e9c776f48938f094bdee22eed66adf8f5a0f57cdf80e8501c542","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"40f98a8fe2ee5769f2f3fe657a600eba6f90c74c363bb5bb349a9ae73cc7816e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"410b9389ac03d165130e802d8c3cdfaec8546dee26ac9e77090a402cc716e054","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"4d15f4dfdd2e58e51237555b35bc63600ea314e59343dbcfe31341db234aa859","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"533cbac3ec6be96bb5072a542410db60f8aab4366bfbbff9e6ab5b55066c1d5d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"5349761d55e1d3b4f54fb24a03e78f21149730ff6a3bcb25874bcd358b26c885","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"55612e7e60aa5b214a343ec878b11b799f3ad15228bb762f6f2d42f273d58f8c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"636460635c5bc4286ba2d967ca22e0e23b6b0e047c29f5fb9c0460bd5027ab47","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"6a70d6a102c3d67539b0a85dd97ef0f7d4875dfd553445f82797dd99de495762","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"6cc23d31934165cf34dc32009b6ff0c069693722df2dd1feea55f38c801b2288","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"6d6808d057091e810fbd2cffa6748da09cf1ab8fcb9cd1e21499cf0bb9d6d89f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"6d6fba57278922b4cd3d4ae33b1a3dccd60667684616d769ac5698eabd1775cf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"71fa4daeddf941d959f8a5e7436bfa6ae16ea764292a86d6a5ddf60b132b49ef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"758aaae83e0a9fd755bb48e054201f5460f0f19cc2e94df9eab93a2beada0d25","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"793c9d44ab88946c370b5371d83289174c3890e70a4495861563c7efa748f749","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"84e37b2ada7bf4aacc5da593356f968ad671fcbad815be3ff029c179d4e70411","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"876a3197e143a50d5b65d13c8c09799475bb5402159370b150dc52035d542657","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"98848efa726d399117c2fd38aefbdafdc74561ebaf221abd6f86c0454095723a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"995e2333d4b71119df0c43ad4e61198a20ea720f67a0f3e337581fd90baff684","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"9aecf61eb5a9f9a975e5aa14f39a90bdfc38ac523058bcae8f9bf4dad89c51df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"a22a53e8f9a20a64c491333b1486075bd0f09e83eb105d01787348d84d3680f0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"adcb37889e62838134af124399c06ce3a8853c539af8b797cd7f8bbc79adb5b1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"b5bbd507925ef8880bb5f5324eca49f058e039e9ba23da80117b4b33696baf12","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"b5ecba9dbcb5f6799c7849b3c68905933a4a66a18831e4a23305eb62109932e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"b775a05d51c34270a80137e04f37d044984f99e0b585f5c6325dd060a0445c5c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"b94b7ba3fe669c5bc9e337f91cd0851ce2b2431a762139f95992782056ec5752","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"bb881a23506bae75af29bb108e4b187d3c077919067f25560736d3f9fbfdbac2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"c5e8457b43f70984bf5e02b577526f9f57ca7abbc1b2e88a15d8e6402b2d9b54","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"dcc962ab743e58b2b00783347a59e279647df2aea8105b5c97d8cb8e7458de06","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"df9d44464abdc3c3a029944c2955eff3ef6d3edb2e2a6f7d195aac8b5bbf1e81","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"fa3f9bf16be572d3463b45f36846100999ac8349178ce130103c33e88b7f8cde","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2075716817757671424","occurred_at":"2026-07-10T23:00:28Z","signature_id":"17d4ef6a848ccbd9a1c3983904d707817e3a3eedce0da626c102274b96c1db15","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"0003cabe51c1c01edcb2207e546b38cad1eae96c99e9cecf2d3c6f64b39be3e3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"094d5f5e30b5ea4bc3960153778f689f00351509fed8c4d99d6d7fb1ec9aed55","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"108edf565c7eca2464b51bdfbc106c93d811e429e473079818a5537222fa5e50","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"23bf1d14de43c5f95058a5b8512d2cfd9a1e696f5d43b77ed656b42a8f36074b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"25adc7e64dc1a3956a205646cabb071643e30520bbee959dc1ddba8534183c9d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"27ef1864ed49017a7e216dbb60d6b5ea8054a563b1a670789c285ba87e3c7297","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"297cfe1effc1bd34d70033dd09ac971dd452fc5845964218cd679f4317512385","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"37754912d4b084426e70c9a109d25bc03a70fab94fd38dc2ad8209fbd140d07d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"63a4f0336eb18341eefc1311f6c2cda4f917468e46c216049b76e146b4f628eb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"659af3978e8b63e1b66c619018abba7a5891eeede1fc839f579cba93cb496284","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"6d3c114015f23efa33a47d41dc04e8b4dc6a37c2de0dbc3d88a60d9602e1d297","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"7070e25b198c3dca364d18b9267309f841f52e3f6fc1815b211ccc0d000ff17a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"71d72283fa52a0c6b3ef69aa6389b5128c152564a3818742b154635f6161b701","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"79bf39ab5f7656b87e5f04884f042cbe7ddb1b4cf75b0c5b22c2a032d2883439","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"82a7d8496376bc37f8f7e05b26f76c937005216b454b654b70d4833a71b14548","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"97283cff250d9b63f671f55c678d424bc068adb9a64d9850cc29b6af2dc1f40e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"9a572401b3ff166dbcf39dc856299ae13d6ec7d955f0f77678ad3f1ffb9939cd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"a13d92c755b8f153d4c9924b655977660ce8d9ad3c8832d886c74d3763c61a07","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"a941afb00367f2d9cf5d057af6330b06d2e8da2f1f0eaac31fbec1c9bdce7fcb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"b768211abdca0d09d14f4f2a0125fce6e4199cf7faea8c048fa3652cb3b70de4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"bf6a575952021000e0bf74b46fb7a94fef94cae021c743a060425653be86e6a5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"d7e2bb0b213890c4ece1b37b7e025da6bc5690d9f9c38e40d30e5677ed395ee3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"dce4bd90df80eaeebeec2fd0c8f8e80ce6374b5efd61d38a15b5c398f9809e8d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"de14097e126d959dcae731a9f5484116a319647b09e278ea751efda382648b38","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"e430d5414e4a522a155d84bafff228baf8477cb11a46be2a3c89ef3521f8fc2a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"e5de09db273887caf7fcf0d2a52d3e1801ad10d951894bb9aaa19ef4f5b5e140","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"ef684e3e4bc592f98641ceff93349beb5e29367e04c57f0fc902dbdc6d8617a6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"efd93abd2dfd31d79e2528d9276753535ec8125a323e05799fc422d14c41b6e8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"f9a4a62075686e33ee6e8400132df86ceb04e7e97b495667ce94c4f21d77e85c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075180010410348544","occurred_at":"2026-07-09T11:27:24Z","signature_id":"7cffeee611fd8b4f3969a8e8b0497e10b9f0e8d4fac503ed8416732936b147d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075134802771578880","occurred_at":"2026-07-09T08:27:45Z","signature_id":"c2c76af9af1934d2e66d44150bf16a834c6b78d4396ed010bf1c5ecea8d2ca7b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"051cd11894b5bb1761f42390833b9dfa4b35559010eb57af3a4752028484c08e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"0554e6ab0e6a614036a6bcbbbd203e4f4bb3b25ca9dcd4d9ab0f726bda20b283","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"0c8f8984d19fca5e2109d89426e8465f02c21a95d245e2bfffecaacafe077a2a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"15bbf6f42543fdf7cfcb73fae3e921ede323286b5e327c79169ed8f66e8f4119","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"1dbe28aaa1123f83802f1de707b5da7fc2853f3767e0a95ed577eaed89787464","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"1ece47042c09ec8f35bdf130d7465d5968a130e2c070435d1837cee337207908","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"2297825ff7c934f1a0949564961ae25dba26cc537022d08ba17556c8f0143842","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"23a2fa0bad2ec2e2cb932442d27b04f8e284933fd3cd9c6ac247daf6683b23c9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"28e8b642415a41acf0353096baf5f7d28fecfdc10c57942eb496505bf86fa40a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"29b36ea550191b186760d9cc3d69010148ad56d86051a11c874f2cdd88fef386","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"2d8d96f15913cd8c77c68593081a75d4522169b27e23a89f5029858d7d3efd91","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"31e93372abd82a72fc3f31e83198cc64b075fc70d727f16d56929fee1343e2fa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"39a10aab7e47fd4e1832359e1be722185297231ff6b9d34348c591995b419911","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"438e6e3b40fc91b239b897d30af29235d11b1c36777b26c6539b8ce1e59c2bf2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"4627af1e23c7ebdc066d9a600aebc5ab11f5a83fe29e8960f9435305e05b7490","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"472762e9ab805bceb9d643d5e2d3af71ae642684b01bd95e5061e899dc465a00","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"4d09d877cb0ecc01ce0e481f52391481a408a35ec842dd40a7ebe3be0bdedbf2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"5133a9a3510c736dff1ba567230f495eb4fc871d89503d306c44984e205946f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"6849a249e6c2ebcce32d57465d8d5de70673fee2796ccc6549b8f68949d8a30f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"68cb4ec713f1e549306c75d11922bfcf13e13c6a72d65cec267dad29412315c3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"7e24a60a3aaaaeec020d6593623c0904a5a964308294927373b3eb42b40b158b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"898093075b4a8bc027b4d5ed45992df6c20fed3b2737cdbc9c5ad690668b9a47","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"8d2fefd28378eea263a122e791f5bf24be54b891eac45c0b9828883a6d2bd4ca","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"8f71bc7117807f930e17f3b000d46ab0054b7b803aafe1b4ee8186c3546bd095","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"9900df039603a54ab01e168fa49161f5f25e123cb747c63234da923a4825d437","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"9f2ac729fe9d976da65402072c5ce163aec86e2fc9dd555a29c79a9026760bcb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"9f354ce5c9797aa06703b2e23cd0fc2fde85c09e8515cedfd2e2657ea7c923aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"a015ba8d7eed4e159361d381519baece2f756c8b01ac5da13c659a761bf7cea9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"a53592d211b24d3b93fc9adc6cda79a9440cbfcccf2a355f61ad6d89d25aeb7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"a9d3a543b7b2dfdd016d74d3237526fcf910d2eb7f51b885db44ff04acfd21aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"afb4b21aad6687d1326697d4b7a71abb51ae46583ac23b65d65e061a3fff86fe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"b1890c93497df6ed0faf59443ecdf52d709a1cd29a82ebdff4355fe10f069cc9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"b43dbfa46db277cbbecd4363e137ad8177c0506fabbad6467bc1628c34e1f2af","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"c44be3b2c23e0b21e01640dee289704e25248b5de84c0d9af6a9ef4e3622863c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"d1534c2f02127482bc57486a7ebe25aa656a4bb7299fc9040490863de324219b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"d53160db757538538a7bb1d0f9f58b70bdbe22b7f497589b5857d723c14d6029","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"d85ae202e1ac677177b74e124a438429fcb961efc8ca69cecbd30c4eaf4703c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"f915e6b02586f8161979be107e9090aa6286ef6a76578157f3f92ddd8ab7f30b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","signature_id":"2251af67413ee13c4c5e368b0277dd29224009821a309b46277adf7d0691d92a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","signature_id":"660c2d2ef27db1386f6fb6a6518159647b9c2f13e1b68a6c879e6ada6f83ab2d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","signature_id":"6662436447f7c11e5d01b1451bc685b4b1823ce737043dbbdc68cda120e256cf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","signature_id":"853593d09741e209fe5dec6b4d039fc5839ccc8176bd1fb4cf6294a635dae0e6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","signature_id":"8f7f52b33bfba720232f67996e6746df05c55e1bf868cc452089392898d4dbdd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","signature_id":"9e84f1efe606a19589ecd84d8735e48bdf3cae74d15d58520ee2fa46791d6ad0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","signature_id":"c39b6ee185c377797cc31e99dd8d5b2d1c19a4bf52465ead53e538125a379ca6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-p92jfl, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-p92jfl\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":8,"runs_affected":2,"failed_runs":2,"run_impact_percent":7.142857142857143,"anchor_occurrences":8,"after_last_push_count":8,"trend_counts":[0,0,0,5,0,3,0,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":3,"prior_window_starts":["2026-06-08","2026-06-15","2026-06-22"],"prior_runs_affected":34,"prior_last_seen_at":"2026-06-23T01:55:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-s2txmj\" for cluster \"cluster-candidate-4-21-s2txmj\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001b15ec0\u003e: \n    failed to create NodePool np-s2txmj, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-s2txmj in resource group rg-candidate-4-21-hlz6pz, error: failed waiting for nodepool=\"np-s2txmj\" for cluster \"cluster-candidate-4-21-s2txmj\" in resourcegroup=\"rg-candidate-4-21-hlz6pz\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-s2txmj\" for cluster \"cluster-candidate-4-21-s2txmj\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001b15ec0\u003e: \n    failed to create NodePool np-s2txmj, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-s2txmj in resource group rg-candidate-4-21-hlz6pz, error: failed waiting for nodepool=\"np-s2txmj\" for cluster \"cluster-candidate-4-21-s2txmj\" in resourcegroup=\"rg-candidate-4-21-hlz6pz\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-sfgvlw\" for cluster \"cluster-candidate-5-0-sfgvlw\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00030d050\u003e: \n    failed to create NodePool np-sfgvlw, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-sfgvlw in resource group rg-candidate-5-0-dl7nwv, error: failed waiting for nodepool=\"np-sfgvlw\" for cluster \"cluster-candidate-5-0-sfgvlw\" in resourcegroup=\"rg-candidate-5-0-dl7nwv\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-sfgvlw\" for cluster \"cluster-candidate-5-0-sfgvlw\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00030d050\u003e: \n    failed to create NodePool np-sfgvlw, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-sfgvlw in resource group rg-candidate-5-0-dl7nwv, error: failed waiting for nodepool=\"np-sfgvlw\" for cluster \"cluster-candidate-5-0-sfgvlw\" in resourcegroup=\"rg-candidate-5-0-dl7nwv\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:259]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002488cc0\u003e: \n    failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-8k9j5p, error: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-8k9j5p\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:259]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002488cc0\u003e: \n    failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-8k9j5p, error: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-8k9j5p\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"0b5027f5de34123446ebe5f45a2f704fb24995ace2cc881f916a0947a5ed0357","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"1142b94be10c149891bc7663ac8d0a8ff146939fd5db89bf7f3b17aa55bd07b8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"52aa2ed9632a5196773fdccfac3ac521f9c3090d5de6ac41965e72a91450bdf5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","signature_id":"992e661009bd6cc777dff7be4b9f18543ed54486ef1878bb220c1a3eeae9de6a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","signature_id":"9e3ad48475bba3ac4b5df6b42687239fbe9ee4e76afdd8a7ba5e5b0c0c72354f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","signature_id":"a5ae3383c025c2472192d42e693602e44ec17318c3c5c8d1ba5ddaa717587716","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","signature_id":"a9d80ca86d6c087aef0d0dc940b389dc6271865d2787e0f0b8354ec151a74212","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","signature_id":"ee0a3501c1a0d5e64f3e9950f8f901c979e4a1e3b15484eeeca1022d09da8d6e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"admin-credential-lifecycle-test-glprsp\", caused by: failed waiting for hcpCluster=\"admin-cred-lifecycle-n7dbhc\" in resourcegroup=\"admin-credential-lifecycle-test-glprsp\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-n7dbhc in resource group admin-credential-lifecycle-test-glprsp, error: context deadline exceeded, error: failed waiting for hcpCluster=\"admin-cred-lifecycle-n7dbhc\" in resourcegroup=\"admin-credential-lifecycle-test-glprsp\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-n7dbhc in resource group admin-credential-lifecycle-test-glprsp, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":6,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":6,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,6,0,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc002730108\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-skip-minor-j5x5g5-c9l8qq\", caused by: failed waiting for hcpCluster=\"np-skip-minor-j5x5g5\" in resourcegroup=\"rg-np-skip-minor-j5x5g5-c9l8qq\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-j5x5g5 in resource group rg-np-skip-minor-j5x5g5-c9l8qq, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-skip-minor-j5x5g5\" in resourcegroup=\"rg-np-skip-minor-j5x5g5-c9l8qq\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-j5x5g5 in resource group rg-np-skip-minor-j5x5g5-c9l8qq, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc002730108\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-skip-minor-j5x5g5-c9l8qq\", caused by: failed waiting for hcpCluster=\"np-skip-minor-j5x5g5\" in resourcegroup=\"rg-np-skip-minor-j5x5g5-c9l8qq\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-j5x5g5 in resource group rg-np-skip-minor-j5x5g5-c9l8qq, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-skip-minor-j5x5g5\" in resourcegroup=\"rg-np-skip-minor-j5x5g5-c9l8qq\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-j5x5g5 in resource group rg-np-skip-minor-j5x5g5-c9l8qq, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010aa120\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"admin-credential-lifecycle-test-glprsp\", caused by: failed waiting for hcpCluster=\"admin-cred-lifecycle-n7dbhc\" in resourcegroup=\"admin-credential-lifecycle-test-glprsp\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-n7dbhc in resource group admin-credential-lifecycle-test-glprsp, error: context deadline exceeded, error: failed waiting for hcpCluster=\"admin-cred-lifecycle-n7dbhc\" in resourcegroup=\"admin-credential-lifecycle-test-glprsp\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-n7dbhc in resource group admin-credential-lifecycle-test-glprsp, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010aa120\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"admin-credential-lifecycle-test-glprsp\", caused by: failed waiting for hcpCluster=\"admin-cred-lifecycle-n7dbhc\" in resourcegroup=\"admin-credential-lifecycle-test-glprsp\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-n7dbhc in resource group admin-credential-lifecycle-test-glprsp, error: context deadline exceeded, error: failed waiting for hcpCluster=\"admin-cred-lifecycle-n7dbhc\" in resourcegroup=\"admin-credential-lifecycle-test-glprsp\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-n7dbhc in resource group admin-credential-lifecycle-test-glprsp, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc001f842d0\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-wkj8vb\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-wkj8vb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-wkj8vb, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-wkj8vb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-wkj8vb, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc001f842d0\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-wkj8vb\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-wkj8vb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-wkj8vb, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-wkj8vb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-wkj8vb, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2075716817757671424","occurred_at":"2026-07-10T23:00:28Z","signature_id":"207329864642edc3f0c3d96e750c0643f470b9f6d35abc76bb44275dcecb7618","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2075716817757671424","occurred_at":"2026-07-10T23:00:28Z","signature_id":"3d9339b76e37c4f798a9c837c2bedc53796f83f95f3d29d5825aa3ad5a44a388","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2075716817757671424","occurred_at":"2026-07-10T23:00:28Z","signature_id":"57087a845372e6faa22a805ca4567c088ce184b3f64eda5f3774d6aa226913be","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2075716817757671424","occurred_at":"2026-07-10T23:00:28Z","signature_id":"91b2d6e4c499e4d9e6bc34c0a15f3523eaba9a94fb7c5fb87a4697d191b524c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2075716817757671424","occurred_at":"2026-07-10T23:00:28Z","signature_id":"eba1a221dd0bd497896302805295877c144206e4d24929d6412685279740852c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2075716817757671424","occurred_at":"2026-07-10T23:00:28Z","signature_id":"ec6a246836da64ca449ea7b32a8bf2bbdb462b1cacf500755fc659230f0f34e6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"7aebc21de77cb64a5180833ea520c7de32315e635ee94e975fa51f1b8f3df2fd","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.fips-enabled\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":17.857142857142858,"anchor_occurrences":5,"after_last_push_count":9,"trend_counts":[2,2,2,1,0,0,0,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-29"],"prior_runs_affected":9,"prior_last_seen_at":"2026-07-05T23:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008eab80\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-c67zkg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-c67zkg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.fips-enabled\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.fips-enabled]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008eab80\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-c67zkg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-c67zkg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.fips-enabled\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.fips-enabled]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0018a7a40\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-r42t7t\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-r42t7t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.fips-enabled\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.fips-enabled]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0018a7a40\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-r42t7t\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-r42t7t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.fips-enabled\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.fips-enabled]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008bf9e0\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-brwgxc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-brwgxc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.fips-enabled\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.fips-enabled]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008bf9e0\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-brwgxc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-brwgxc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.fips-enabled\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.fips-enabled]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2074674924374462464","occurred_at":"2026-07-08T02:00:22Z","signature_id":"0b80769ddd1b37ce9bab26af28e6639d9ac29c535c4dfdaa44448a9369ea2146","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2074629625081761792","occurred_at":"2026-07-07T23:00:21Z","signature_id":"f097098d1c8c01b41b91c2cf61e523e1e494299d509f22917a967aa4fcd7b93e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2074312532201312256","occurred_at":"2026-07-07T02:00:21Z","signature_id":"f57b27a239688667fb038d49fa9fc606e2fbb99bb1692f744c025829dcb1af69","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2074267232472403968","occurred_at":"2026-07-06T23:00:20Z","signature_id":"a04662048ad40d82b374a283d775eaea0f6beeaff9d3a4ad9323ed132f5d727d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2073950128078589952","occurred_at":"2026-07-06T02:00:17Z","signature_id":"d49724923c85468268d5ab4fc0344b342faf411176bf03ccdd1dccec9462db8b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"7ac9ea1609c23f13fc59d25bc0226d6ddbb91e9217b1f67c5624f53eb77e8333","failure_pattern":"error should indicate FIPS value must be true or false","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:203]: error should indicate FIPS value must be true or false","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should reject an HCP cluster creation with invalid FIPS tag value","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":17.857142857142858,"anchor_occurrences":5,"after_last_push_count":9,"trend_counts":[2,2,2,1,0,0,0,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-29"],"prior_runs_affected":9,"prior_last_seen_at":"2026-07-05T23:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should reject an HCP cluster creation with invalid FIPS tag value","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should reject an HCP cluster creation with invalid FIPS tag value","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:203]: error should indicate FIPS value must be true or false\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:203]: error should indicate FIPS value must be true or false\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2074674924374462464","occurred_at":"2026-07-08T02:00:22Z","signature_id":"de8314c475995e0785e91e95c202ce3fffed3fe3c3aef8af44fc920a56292561","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2074629625081761792","occurred_at":"2026-07-07T23:00:21Z","signature_id":"de8314c475995e0785e91e95c202ce3fffed3fe3c3aef8af44fc920a56292561","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2074312532201312256","occurred_at":"2026-07-07T02:00:21Z","signature_id":"de8314c475995e0785e91e95c202ce3fffed3fe3c3aef8af44fc920a56292561","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2074267232472403968","occurred_at":"2026-07-06T23:00:20Z","signature_id":"de8314c475995e0785e91e95c202ce3fffed3fe3c3aef8af44fc920a56292561","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2073950128078589952","occurred_at":"2026-07-06T02:00:17Z","signature_id":"de8314c475995e0785e91e95c202ce3fffed3fe3c3aef8af44fc920a56292561","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f27b30bef0ebb941cd79b083d3eb5ff3005b4b85f92582e4349d8748b1eaeb94","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-hbdz7m\" to finish deleting: context canceled","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":5,"runs_affected":2,"failed_runs":2,"run_impact_percent":7.142857142857143,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,0,0,1,0,4,0,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":2,"prior_window_starts":["2026-06-15","2026-06-22"],"prior_runs_affected":7,"prior_last_seen_at":"2026-06-23T23:00:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00169e090\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-hbdz7m\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00169e090\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-hbdz7m\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000a20330\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-8s4rn7\" in resourcegroup=\"rg-zstream-upgrade-4-20-tp4pvb\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000a20330\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-8s4rn7\" in resourcegroup=\"rg-zstream-upgrade-4-20-tp4pvb\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000d4b050\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-rvc9mn\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000d4b050\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-rvc9mn\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"3e5a6c977e959ca69eb1e1f2872b2dc7084d03a1803f8929cfba65b48edbf01c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"58c5fbb1501b065dbc1c9b23e11576aa0fcb1713cea279e948cf9be9d20e61ca","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"eee44a43fcd7a9a471f535f17ae4c45dfbaac3b8bb7f3fc7de2d77974828bba8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"fd64f8d7a67988e152ec5e7c782f9bd72cb8a1a4419ea2fa530aafd1c26d041a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","signature_id":"c7dd31532f3fb1df869fdf3125cd542623e96eb4c34228eb7234b23f46929991","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dcad60eb8db626db10bfb0812a18e7c392049987a81c4fcbcdc5140e0032dc41","failure_pattern":"Cluster should become ready within '20.000000' minutes","search_query":"Cluster should become ready within '20.000000' minutes","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":10.714285714285714,"anchor_occurrences":3,"after_last_push_count":8,"trend_counts":[0,0,0,1,0,1,1,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":3,"prior_window_starts":["2026-06-08","2026-06-15","2026-06-22"],"prior_runs_affected":12,"prior_last_seen_at":"2026-06-23T05:14:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 1200.001s.\nCluster should become ready within '20.000000' minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 1200.001s.\nCluster should become ready within '20.000000' minutes\nExpected\n    \u003cbool\u003e: ...\nto be true","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 1200.004s.\nCluster should become ready within '20.000000' minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 1200.004s.\nCluster should become ready within '20.000000' minutes\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"7808e0d17261c46203073385b1bf48fa70b107abdef9661b60e820b1b8327817","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"7653ffd4e02735b234dd652c0dd32332c9566363ef7806113eb84037d3ada7a9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"7653ffd4e02735b234dd652c0dd32332c9566363ef7806113eb84037d3ada7a9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"cluster-zstream-4-22-9mnhf7\" in resourcegroup=\"rg-zstream-upgrade-4-22-6vc2xq\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-22-9mnhf7 in resource group rg-zstream-upgrade-4-22-6vc2xq, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":10.714285714285714,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,1,0,2,0,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-15"],"prior_runs_affected":4,"prior_last_seen_at":"2026-06-17T10:52:58Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:119]: failed to get admin REST config for HCP cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001cfeba0\u003e: \n    failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-pqvwsl\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster fips-enabled-cluster in resource group fips-enabled-pqvwsl, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:119]: failed to get admin REST config for HCP cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001cfeba0\u003e: \n    failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-pqvwsl\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster fips-enabled-cluster in resource group fips-enabled-pqvwsl, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:119]: failed to get admin REST config for cluster \"cluster-zstream-4-22-9mnhf7\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0005f6cf0\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-22-9mnhf7\" in resourcegroup=\"rg-zstream-upgrade-4-22-6vc2xq\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-22-9mnhf7 in resource group rg-zstream-upgrade-4-22-6vc2xq, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:119]: failed to get admin REST config for cluster \"cluster-zstream-4-22-9mnhf7\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0005f6cf0\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-22-9mnhf7\" in resourcegroup=\"rg-zstream-upgrade-4-22-6vc2xq\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-22-9mnhf7 in resource group rg-zstream-upgrade-4-22-6vc2xq, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:161]: failed to get admin REST config for cluster np-version-upgrade-cluster-gv255d\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002418720\u003e: \n    failed waiting for hcpCluster=\"np-version-upgrade-cluster-gv255d\" in resourcegroup=\"rg-np-version-upgrade-gv255d-bzslj8\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster np-version-upgrade-cluster-gv255d in resource group rg-np-version-upgrade-gv255d-bzslj8, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:161]: failed to get admin REST config for cluster np-version-upgrade-cluster-gv255d\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002418720\u003e: \n    failed waiting for hcpCluster=\"np-version-upgrade-cluster-gv255d\" in resourcegroup=\"rg-np-version-upgrade-gv255d-bzslj8\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster np-version-upgrade-cluster-gv255d in resource group rg-np-version-upgrade-gv255d-bzslj8, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2075716817757671424","occurred_at":"2026-07-10T23:00:28Z","signature_id":"f2a000470a0574bdb22aea8c479c3d131c656065568d206765f6f170da921cde","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075436831297507328","occurred_at":"2026-07-10T04:27:54Z","signature_id":"85d3c395cf9942afe6177cadcf6940fd336434719405e671310968feb2408307","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","signature_id":"a3b7a0d2c3e838c32f8c3c0a0c703c998a7038644320c2a1807d8b3f012cff23","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":10.714285714285714,"anchor_occurrences":3,"after_last_push_count":7,"trend_counts":[1,0,0,0,3,0,0,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-08","2026-06-15","2026-06-22","2026-06-29"],"prior_runs_affected":9,"prior_last_seen_at":"2026-07-05T02:00:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-nightly-4-21-pgpqxt\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00102e060\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-njf6z.apps.aro.d8f0y3s7c3c7p2d.ek0b.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-nightly-4-21-pgpqxt\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00102e060\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-njf6z.apps.aro.d8f0y3s7c3c7p2d.ek0b.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-grqfrx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000498440\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-rvc2j.apps.aro.b6f7d0t5b9c7m1v.opfq.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-grqfrx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000498440\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-rvc2j.apps.aro.b6f7d0t5b9c7m1v.opfq.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:197]: failed to verify simple web app on back-level cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f18c00\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-wtf7m.apps.aro.e8g5g8f5f1b8d7k.ykxw.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:197]: failed to verify simple web app on back-level cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f18c00\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-wtf7m.apps.aro.e8g5g8f5f1b8d7k.ykxw.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2075354410581495808","occurred_at":"2026-07-09T23:00:24Z","signature_id":"ce9271b96f3c7b6923ef0fc23f10cbc3bb253833feb6393b6512b87a6e5385a0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075076265840218112","occurred_at":"2026-07-09T04:26:09Z","signature_id":"2c5e1ef9f86cbc1a3ccd631bec06c4e317d9335ea5cd80a3c32389b04b08c56b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075012973046272000","occurred_at":"2026-07-09T00:23:39Z","signature_id":"83ba665f08c48a739d7428e398ec28fefe20bd01ef4ba7691eee471350635f8d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1141d0fd9f679007e447ee31146ba219b79d56c1f3a9288dd68187b3f9e7036c","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-9qjlw2\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-9qjlw2, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":7.142857142857143,"anchor_occurrences":3,"after_last_push_count":4,"trend_counts":[0,0,0,2,1,0,0,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-15"],"prior_runs_affected":3,"prior_last_seen_at":"2026-06-17T10:52:58Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:185]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0027caba0\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-9qjlw2\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-9qjlw2, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:185]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0027caba0\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-9qjlw2\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-9qjlw2, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:210]: failed to upgrade node pool npupgrade-4-20 to version 4.21.22\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000890ea0\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-fr5x47\" resourcegroup=\"rg-np-version-upgrade-fr5x47-tjnxv8\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-fr5x47 in resource group rg-np-version-upgrade-fr5x47-tjnxv8, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:210]: failed to upgrade node pool npupgrade-4-20 to version 4.21.22\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000890ea0\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-fr5x47\" resourcegroup=\"rg-np-version-upgrade-fr5x47-tjnxv8\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-fr5x47 in resource group rg-np-version-upgrade-fr5x47-tjnxv8, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:185]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0025850e0\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-cqzk85\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-cqzk85, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:185]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0025850e0\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-cqzk85\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-cqzk85, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075239334985863168","occurred_at":"2026-07-09T15:23:08Z","signature_id":"db526c7377214eb629dba3536f3d51a86509e56fb2823a3403d7890ad7f6af71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","signature_id":"abef8e61097f5ada28782f0e139e08fc161b405f6953ce3af4fe771e91e885d3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","signature_id":"f10682ffc40f91f35f9475cee00f4e30cdbfbdecc512a24a5d8f190926e14171","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a1df4a102ffa44f97ddbf6779aa0d2a52a0f6f8f1f86a9bebf09b9cdf20d5052","failure_pattern":"Deserialization Error: no output from command","search_query":"Deserialization Error: no output from command","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":3,"after_last_push_count":9,"trend_counts":[0,0,0,3,0,0,0,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-15"],"prior_runs_affected":6,"prior_last_seen_at":"2026-06-21T07:05:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["c000ea0748)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0xc00158dac8 sp=0xc00158daa0 pc=0x2f5b405\ngolang.org/x/sync/errgroup.(*Group).Wait(0xc000ea0740)\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:56 +0x1e fp=0xc00158dae8 sp=0xc00158dac8 pc=0x4a8aabe\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610({0x7f130446f260, 0xc00157e930}, 0xc000a2af30, {0xc000dc6ea0, 0x22}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:525 +0x8de fp=0xc00158dd90 sp=0xc00158dae8 pc=0x6c8efde\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).cleanupResourceGroup(0xc000e4c840, {0x7f130446f260, 0xc00157e930}, {0xc000dc6ea0, 0x22}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:537 +0x29d fp=0xc00158df08 sp=0xc00158dd90 pc=0x6cb215d\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.func1({0x7f130446f260?, 0xc00157e930?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:360 +0x151 fp=0xc00158dfc0 sp=0xc00158df08 pc=0x6cb01f1\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:369 +0x28 fp=0xc00158dfe0 sp=0xc00158dfc0 pc=0x6cb0068\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc00158dfe8 sp=0xc00158dfe0 pc=0x2f3ecc1\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 517\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:353 +0x98\n\ngoroutine 533 gp=0xc001103340 m=nil [select]:\nruntime.gopark(0xc00189d768?, 0x2?, 0x0?, 0x40?, 0xc00189d75c?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc00189d5e8 sp=0xc00189d5c8 pc=0x2f3650e\nruntime.selectgo(0xc00189d768, 0xc00189d758, 0xc000a80180?, 0x0, 0x370521c?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc00189d728 sp=0xc00189d5e8 pc=0x2f112f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x9376198, 0xc000033490}, 0x8779154?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc00189d798 sp=0xc00189d728 pc=0x3704976\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x93c8b00, {0x9376198, 0xc000033490}, 0x2f33af3)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc00189da98 sp=0xc00189d798 pc=0x3e10172\ngithub.com/Azure/ARO-HCP/test/util/framework.deleteHCPClusterAttempt({0x9376198, 0xc000033490}, 0xc000a2af30, {0xc000dc6ea0, 0x22}, {0xc0008e1ca0, 0xd})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:240 +0x365 fp=0xc00189dc20 sp=0xc00189da98 pc=0x6c8bf85\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610.func1({0x9376198?, 0xc000033490?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:198 +0x72 fp=0xc00189dcc8 sp=0xc00189dc20 pc=0x6c8b9b2\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext({0x9376198?, 0xc000033490?}, 0xc0000334a0?)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:159 +0x62 fp=0xc00189dd20 sp=0xc00189dcc8 pc=0x36b7f42\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoffWithContext({0x9376198, 0xc000033490}, {0xdf8475800, 0x4000000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0xc00189de68)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:501 +0xae fp=0xc00189dd98 sp=0xc00189dd20 pc=0x36b5fee\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610({0x9376128, 0xc000e91b80}, 0xc000a2af30, {0xc000dc6ea0, 0x22}, {0xc0008e1ca0, 0xd}, 0x274a48a7800)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:196 +0x285 fp=0xc00189def0 sp=0xc00189dd98 pc=0x6c8b725\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:522 +0x7e fp=0xc00189df78 sp=0xc00189def0 pc=0x6c8f61e\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc00189dfe0 sp=0xc00189df78 pc=0x4a8ac10\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc00189dfe8 sp=0xc00189dfe0 pc=0x2f3ecc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 526\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\nrax    0xca\nrbx    0x0\nrcx    0x2f40ae3\nrdx    0x0\nrdi    0xc9bba10\nrsi    0x80\nrbp    0x7ffc382442f0\nrsp    0x7ffc382442a8\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x3\nr13    0x1\nr14    0xc9b9480\nr15    0x7\nrip    0x2f40ae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\nc000ea0748)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0xc00158dac8 sp=0xc00158daa0 pc=0x2f5b405\ngolang.org/x/sync/errgroup.(*Group).Wait(0xc000ea0740)\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:56 +0x1e fp=0xc00158dae8 sp=0xc00158dac8 pc=0x4a8aabe\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610({0x7f130446f260, 0xc00157e930}, 0xc000a2af30, {0xc000dc6ea0, 0x22}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:525 +0x8de fp=0xc00158dd90 sp=0xc00158dae8 pc=0x6c8efde\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).cleanupResourceGroup(0xc000e4c840, {0x7f130446f260, 0xc00157e930}, {0xc000dc6ea0, 0x22}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:537 +0x29d fp=0xc00158df08 sp=0xc00158dd90 pc=0x6cb215d\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.func1({0x7f130446f260?, 0xc00157e930?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:360 +0x151 fp=0xc00158dfc0 sp=0xc00158df08 pc=0x6cb01f1\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:369 +0x28 fp=0xc00158dfe0 sp=0xc00158dfc0 pc=0x6cb0068\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc00158dfe8 sp=0xc00158dfe0 pc=0x2f3ecc1\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 517\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:353 +0x98\n\ngoroutine 533 gp=0xc001103340 m=nil [select]:\nruntime.gopark(0xc00189d768?, 0x2?, 0x0?, 0x40?, 0xc00189d75c?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc00189d5e8 sp=0xc00189d5c8 pc=0x2f3650e\nruntime.selectgo(0xc00189d768, 0xc00189d758, 0xc000a80180?, 0x0, 0x370521c?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc00189d728 sp=0xc00189d5e8 pc=0x2f112f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x9376198, 0xc000033490}, 0x8779154?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc00189d798 sp=0xc00189d728 pc=0x3704976\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x93c8b00, {0x9376198, 0xc000033490}, 0x2f33af3)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc00189da98 sp=0xc00189d798 pc=0x3e10172\ngithub.com/Azure/ARO-HCP/test/util/framework.deleteHCPClusterAttempt({0x9376198, 0xc000033490}, 0xc000a2af30, {0xc000dc6ea0, 0x22}, {0xc0008e1ca0, 0xd})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:240 +0x365 fp=0xc00189dc20 sp=0xc00189da98 pc=0x6c8bf85\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610.func1({0x9376198?, 0xc000033490?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:198 +0x72 fp=0xc00189dcc8 sp=0xc00189dc20 pc=0x6c8b9b2\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext({0x9376198?, 0xc000033490?}, 0xc0000334a0?)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:159 +0x62 fp=0xc00189dd20 sp=0xc00189dcc8 pc=0x36b7f42\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoffWithContext({0x9376198, 0xc000033490}, {0xdf8475800, 0x4000000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0xc00189de68)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:501 +0xae fp=0xc00189dd98 sp=0xc00189dd20 pc=0x36b5fee\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610({0x9376128, 0xc000e91b80}, 0xc000a2af30, {0xc000dc6ea0, 0x22}, {0xc0008e1ca0, 0xd}, 0x274a48a7800)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:196 +0x285 fp=0xc00189def0 sp=0xc00189dd98 pc=0x6c8b725\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:522 +0x7e fp=0xc00189df78 sp=0xc00189def0 pc=0x6c8f61e\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc00189dfe0 sp=0xc00189df78 pc=0x4a8ac10\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc00189dfe8 sp=0xc00189dfe0 pc=0x2f3ecc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 526\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\nrax    0xca\nrbx    0x0\nrcx    0x2f40ae3\nrdx    0x0\nrdi    0xc9bba10\nrsi    0x80\nrbp    0x7ffc382442f0\nrsp    0x7ffc382442a8\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x3\nr13    0x1\nr14    0xc9b9480\nr15    0x7\nrip    0x2f40ae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command","6cb01f1\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:369 +0x28 fp=0xc000ecdfe0 sp=0xc000ecdfc0 pc=0x6cb0068\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000ecdfe8 sp=0xc000ecdfe0 pc=0x2f3ecc1\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 282\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:353 +0x98\n\ngoroutine 315 gp=0xc0016b9340 m=nil [select]:\nruntime.gopark(0xc000b05f48?, 0x2?, 0xc0?, 0x5d?, 0xc000b05ef4?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000b05d78 sp=0xc000b05d58 pc=0x2f3650e\nruntime.selectgo(0xc000b05f48, 0xc000b05ef0, 0xc000e26c40?, 0x0, 0x0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000b05eb8 sp=0xc000b05d78 pc=0x2f112f7\nnet/http.(*persistConn).writeLoop(0xc000adefc0)\n\t/usr/lib/golang/src/net/http/transport.go:2600 +0xe7 fp=0xc000b05fc8 sp=0xc000b05eb8 pc=0x32fec87\nnet/http.(*Transport).dialConn.gowrap3()\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x25 fp=0xc000b05fe0 sp=0xc000b05fc8 pc=0x32fb925\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000b05fe8 sp=0xc000b05fe0 pc=0x2f3ecc1\ncreated by net/http.(*Transport).dialConn in goroutine 283\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x17a5\n\ngoroutine 314 gp=0xc0016b96c0 m=nil [IO wait]:\nruntime.gopark(0xc0009cc3a0?, 0xc0000c3788?, 0xc5?, 0x3e?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc0000c3768 sp=0xc0000c3748 pc=0x2f3650e\nruntime.netpollblock(0x2fa0ad8?, 0x2ec2066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc0000c37a0 sp=0xc0000c3768 pc=0x2ef5fb7\ninternal/poll.runtime_pollWait(0x7f0ad6814a00, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc0000c37c0 sp=0xc0000c37a0 pc=0x2f34da5\ninternal/poll.(*pollDesc).wait(0xc001816c80?, 0xc000dec000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc0000c37e8 sp=0xc0000c37c0 pc=0x2fc3907\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc001816c80, {0xc000dec000, 0x1300, 0x1300})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc0000c3880 sp=0xc0000c37e8 pc=0x2fc4d39\nnet.(*netFD).Read(0xc001816c80, {0xc000dec000?, 0xc000dec000?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc0000c38c8 sp=0xc0000c3880 pc=0x31893c5\nnet.(*conn).Read(0xc0000b4318, {0xc000dec000?, 0x7f0ad57c8a48?, 0x7f0b1d3fb5c0?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc0000c3910 sp=0xc0000c38c8 pc=0x319b465\ncrypto/tls.(*atLeastReader).Read(0xc0010f29d8, {0xc000dec000?, 0x120a?, 0x2ed79d6?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc0000c3958 sp=0xc0000c3910 pc=0x3206ffb\nbytes.(*Buffer).ReadFrom(0xc0009a82a8, {0x92dd8a0, 0xc0010f29d8})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc0000c39b0 sp=0xc0000c3958 pc=0x3007578\ncrypto/tls.(*Conn).readFromUntil(0xc0009a8008, {0x92df380, 0xc0000b4318}, 0xc0000c3a18?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc0000c39e8 sp=0xc0000c39b0 pc=0x32071de\ncrypto/tls.(*Conn).readRecordOrCCS(0xc0009a8008, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc0000c3c60 sp=0xc0000c39e8 pc=0x320425b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc0009a8008, {0xc000b0e000, 0x1000, 0xc000a4f650?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc0000c3cc8 sp=0xc0000c3c60 pc=0x320ab65\nnet/http.(*persistConn).Read(0xc000adefc0, {0xc000b0e000?, 0x32f4be5?, 0x92d8260?})\n\t/usr/lib/golang/src/net/http/transport.go:2125 +0x47 fp=0xc0000c3d28 sp=0xc0000c3cc8 pc=0x32fc4e7\nbufio.(*Reader).fill(0xc000c2daa0)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0xc0000c3d60 sp=0xc0000c3d28 pc=0x309ca23\nbufio.(*Reader).Peek(0xc000c2daa0, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0xc0000c3d80 sp=0xc0000c3d60 pc=0x309cbb3\nnet/http.(*persistConn).readLoop(0xc000adefc0)\n\t/usr/lib/golang/src/net/http/transport.go:2278 +0x172 fp=0xc0000c3fc8 sp=0xc0000c3d80 pc=0x32fcff2\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x25 fp=0xc0000c3fe0 sp=0xc0000c3fc8 pc=0x32fb985\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000c3fe8 sp=0xc0000c3fe0 pc=0x2f3ecc1\ncreated by net/http.(*Transport).dialConn in goroutine 283\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x174f\n\nrax    0xca\nrbx    0x0\nrcx    0x2f40ae3\nrdx    0x0\nrdi    0xc9bba10\nrsi    0x80\nrbp    0x7fff7dbe62d0\nrsp    0x7fff7dbe6288\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x0\nr13    0x0\nr14    0xc9b9480\nr15    0x7\nrip    0x2f40ae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\n6cb01f1\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:369 +0x28 fp=0xc000ecdfe0 sp=0xc000ecdfc0 pc=0x6cb0068\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000ecdfe8 sp=0xc000ecdfe0 pc=0x2f3ecc1\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 282\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:353 +0x98\n\ngoroutine 315 gp=0xc0016b9340 m=nil [select]:\nruntime.gopark(0xc000b05f48?, 0x2?, 0xc0?, 0x5d?, 0xc000b05ef4?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000b05d78 sp=0xc000b05d58 pc=0x2f3650e\nruntime.selectgo(0xc000b05f48, 0xc000b05ef0, 0xc000e26c40?, 0x0, 0x0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000b05eb8 sp=0xc000b05d78 pc=0x2f112f7\nnet/http.(*persistConn).writeLoop(0xc000adefc0)\n\t/usr/lib/golang/src/net/http/transport.go:2600 +0xe7 fp=0xc000b05fc8 sp=0xc000b05eb8 pc=0x32fec87\nnet/http.(*Transport).dialConn.gowrap3()\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x25 fp=0xc000b05fe0 sp=0xc000b05fc8 pc=0x32fb925\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000b05fe8 sp=0xc000b05fe0 pc=0x2f3ecc1\ncreated by net/http.(*Transport).dialConn in goroutine 283\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x17a5\n\ngoroutine 314 gp=0xc0016b96c0 m=nil [IO wait]:\nruntime.gopark(0xc0009cc3a0?, 0xc0000c3788?, 0xc5?, 0x3e?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc0000c3768 sp=0xc0000c3748 pc=0x2f3650e\nruntime.netpollblock(0x2fa0ad8?, 0x2ec2066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc0000c37a0 sp=0xc0000c3768 pc=0x2ef5fb7\ninternal/poll.runtime_pollWait(0x7f0ad6814a00, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc0000c37c0 sp=0xc0000c37a0 pc=0x2f34da5\ninternal/poll.(*pollDesc).wait(0xc001816c80?, 0xc000dec000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc0000c37e8 sp=0xc0000c37c0 pc=0x2fc3907\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc001816c80, {0xc000dec000, 0x1300, 0x1300})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc0000c3880 sp=0xc0000c37e8 pc=0x2fc4d39\nnet.(*netFD).Read(0xc001816c80, {0xc000dec000?, 0xc000dec000?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc0000c38c8 sp=0xc0000c3880 pc=0x31893c5\nnet.(*conn).Read(0xc0000b4318, {0xc000dec000?, 0x7f0ad57c8a48?, 0x7f0b1d3fb5c0?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc0000c3910 sp=0xc0000c38c8 pc=0x319b465\ncrypto/tls.(*atLeastReader).Read(0xc0010f29d8, {0xc000dec000?, 0x120a?, 0x2ed79d6?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc0000c3958 sp=0xc0000c3910 pc=0x3206ffb\nbytes.(*Buffer).ReadFrom(0xc0009a82a8, {0x92dd8a0, 0xc0010f29d8})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc0000c39b0 sp=0xc0000c3958 pc=0x3007578\ncrypto/tls.(*Conn).readFromUntil(0xc0009a8008, {0x92df380, 0xc0000b4318}, 0xc0000c3a18?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc0000c39e8 sp=0xc0000c39b0 pc=0x32071de\ncrypto/tls.(*Conn).readRecordOrCCS(0xc0009a8008, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc0000c3c60 sp=0xc0000c39e8 pc=0x320425b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc0009a8008, {0xc000b0e000, 0x1000, 0xc000a4f650?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc0000c3cc8 sp=0xc0000c3c60 pc=0x320ab65\nnet/http.(*persistConn).Read(0xc000adefc0, {0xc000b0e000?, 0x32f4be5?, 0x92d8260?})\n\t/usr/lib/golang/src/net/http/transport.go:2125 +0x47 fp=0xc0000c3d28 sp=0xc0000c3cc8 pc=0x32fc4e7\nbufio.(*Reader).fill(0xc000c2daa0)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0xc0000c3d60 sp=0xc0000c3d28 pc=0x309ca23\nbufio.(*Reader).Peek(0xc000c2daa0, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0xc0000c3d80 sp=0xc0000c3d60 pc=0x309cbb3\nnet/http.(*persistConn).readLoop(0xc000adefc0)\n\t/usr/lib/golang/src/net/http/transport.go:2278 +0x172 fp=0xc0000c3fc8 sp=0xc0000c3d80 pc=0x32fcff2\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x25 fp=0xc0000c3fe0 sp=0xc0000c3fc8 pc=0x32fb985\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000c3fe8 sp=0xc0000c3fe0 pc=0x2f3ecc1\ncreated by net/http.(*Transport).dialConn in goroutine 283\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x174f\n\nrax    0xca\nrbx    0x0\nrcx    0x2f40ae3\nrdx    0x0\nrdi    0xc9bba10\nrsi    0x80\nrbp    0x7fff7dbe62d0\nrsp    0x7fff7dbe6288\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x0\nr13    0x0\nr14    0xc9b9480\nr15    0x7\nrip    0x2f40ae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command","ync/waitgroup.go:206 +0x85 fp=0xc000e5fac8 sp=0xc000e5faa0 pc=0x2f5b405\ngolang.org/x/sync/errgroup.(*Group).Wait(0xc000e97840)\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:56 +0x1e fp=0xc000e5fae8 sp=0xc000e5fac8 pc=0x4a8aabe\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610({0x7fc97c1bf660, 0xc000b97f50}, 0xc001082750, {0xc0009bb5c0, 0x11}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:525 +0x8de fp=0xc000e5fd90 sp=0xc000e5fae8 pc=0x6c8efde\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).cleanupResourceGroup(0xc000c154a0, {0x7fc97c1bf660, 0xc000b97f50}, {0xc0009bb5c0, 0x11}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:537 +0x29d fp=0xc000e5ff08 sp=0xc000e5fd90 pc=0x6cb215d\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.func1({0x7fc97c1bf660?, 0xc000b97f50?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:360 +0x151 fp=0xc000e5ffc0 sp=0xc000e5ff08 pc=0x6cb01f1\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:369 +0x28 fp=0xc000e5ffe0 sp=0xc000e5ffc0 pc=0x6cb0068\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000e5ffe8 sp=0xc000e5ffe0 pc=0x2f3ecc1\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 470\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:353 +0x98\n\ngoroutine 491 gp=0xc0011ae380 m=nil [select]:\nruntime.gopark(0xc00165b768?, 0x2?, 0x0?, 0x5a?, 0xc00165b75c?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc00165b5e8 sp=0xc00165b5c8 pc=0x2f3650e\nruntime.selectgo(0xc00165b768, 0xc00165b758, 0xc0006feca0?, 0x0, 0x370521c?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc00165b728 sp=0xc00165b5e8 pc=0x2f112f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x9376198, 0xc001566070}, 0x8779154?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc00165b798 sp=0xc00165b728 pc=0x3704976\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x93c8b00, {0x9376198, 0xc001566070}, 0x2f33af3)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc00165ba98 sp=0xc00165b798 pc=0x3e10172\ngithub.com/Azure/ARO-HCP/test/util/framework.deleteHCPClusterAttempt({0x9376198, 0xc001566070}, 0xc001082750, {0xc0009bb5c0, 0x11}, {0xc00154a300, 0x12})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:240 +0x365 fp=0xc00165bc20 sp=0xc00165ba98 pc=0x6c8bf85\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610.func1({0x9376198?, 0xc001566070?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:198 +0x72 fp=0xc00165bcc8 sp=0xc00165bc20 pc=0x6c8b9b2\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext({0x9376198?, 0xc001566070?}, 0xc001566080?)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:159 +0x62 fp=0xc00165bd20 sp=0xc00165bcc8 pc=0x36b7f42\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoffWithContext({0x9376198, 0xc001566070}, {0xdf8475800, 0x4000000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0xc00165be68)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:501 +0xae fp=0xc00165bd98 sp=0xc00165bd20 pc=0x36b5fee\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610({0x9376128, 0xc0007baff0}, 0xc001082750, {0xc0009bb5c0, 0x11}, {0xc00154a300, 0x12}, 0x274a48a7800)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:196 +0x285 fp=0xc00165bef0 sp=0xc00165bd98 pc=0x6c8b725\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:522 +0x7e fp=0xc00165bf78 sp=0xc00165bef0 pc=0x6c8f61e\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc00165bfe0 sp=0xc00165bf78 pc=0x4a8ac10\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc00165bfe8 sp=0xc00165bfe0 pc=0x2f3ecc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 502\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\nrax    0xfffffffffffffffc\nrbx    0x5\nrcx    0x2eb5e2e\nrdx    0x80\nrdi    0x5\nrsi    0x7ffcb9d27d24\nrbp    0x7ffcb9d27cd8\nrsp    0x7ffcb9d27c90\nr8     0x0\nr9     0x0\nr10    0x393\nr11    0x246\nr12    0x7ffcb9d27d68\nr13    0x1\nr14    0xc9b9480\nr15    0x3\nrip    0x2eb5e2e\nrflags 0x246\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\nync/waitgroup.go:206 +0x85 fp=0xc000e5fac8 sp=0xc000e5faa0 pc=0x2f5b405\ngolang.org/x/sync/errgroup.(*Group).Wait(0xc000e97840)\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:56 +0x1e fp=0xc000e5fae8 sp=0xc000e5fac8 pc=0x4a8aabe\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610({0x7fc97c1bf660, 0xc000b97f50}, 0xc001082750, {0xc0009bb5c0, 0x11}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:525 +0x8de fp=0xc000e5fd90 sp=0xc000e5fae8 pc=0x6c8efde\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).cleanupResourceGroup(0xc000c154a0, {0x7fc97c1bf660, 0xc000b97f50}, {0xc0009bb5c0, 0x11}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:537 +0x29d fp=0xc000e5ff08 sp=0xc000e5fd90 pc=0x6cb215d\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.func1({0x7fc97c1bf660?, 0xc000b97f50?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:360 +0x151 fp=0xc000e5ffc0 sp=0xc000e5ff08 pc=0x6cb01f1\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:369 +0x28 fp=0xc000e5ffe0 sp=0xc000e5ffc0 pc=0x6cb0068\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000e5ffe8 sp=0xc000e5ffe0 pc=0x2f3ecc1\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 470\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:353 +0x98\n\ngoroutine 491 gp=0xc0011ae380 m=nil [select]:\nruntime.gopark(0xc00165b768?, 0x2?, 0x0?, 0x5a?, 0xc00165b75c?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc00165b5e8 sp=0xc00165b5c8 pc=0x2f3650e\nruntime.selectgo(0xc00165b768, 0xc00165b758, 0xc0006feca0?, 0x0, 0x370521c?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc00165b728 sp=0xc00165b5e8 pc=0x2f112f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x9376198, 0xc001566070}, 0x8779154?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc00165b798 sp=0xc00165b728 pc=0x3704976\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x93c8b00, {0x9376198, 0xc001566070}, 0x2f33af3)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc00165ba98 sp=0xc00165b798 pc=0x3e10172\ngithub.com/Azure/ARO-HCP/test/util/framework.deleteHCPClusterAttempt({0x9376198, 0xc001566070}, 0xc001082750, {0xc0009bb5c0, 0x11}, {0xc00154a300, 0x12})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:240 +0x365 fp=0xc00165bc20 sp=0xc00165ba98 pc=0x6c8bf85\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610.func1({0x9376198?, 0xc001566070?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:198 +0x72 fp=0xc00165bcc8 sp=0xc00165bc20 pc=0x6c8b9b2\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext({0x9376198?, 0xc001566070?}, 0xc001566080?)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:159 +0x62 fp=0xc00165bd20 sp=0xc00165bcc8 pc=0x36b7f42\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoffWithContext({0x9376198, 0xc001566070}, {0xdf8475800, 0x4000000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0xc00165be68)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:501 +0xae fp=0xc00165bd98 sp=0xc00165bd20 pc=0x36b5fee\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610({0x9376128, 0xc0007baff0}, 0xc001082750, {0xc0009bb5c0, 0x11}, {0xc00154a300, 0x12}, 0x274a48a7800)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:196 +0x285 fp=0xc00165bef0 sp=0xc00165bd98 pc=0x6c8b725\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:522 +0x7e fp=0xc00165bf78 sp=0xc00165bef0 pc=0x6c8f61e\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc00165bfe0 sp=0xc00165bf78 pc=0x4a8ac10\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc00165bfe8 sp=0xc00165bfe0 pc=0x2f3ecc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 502\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\nrax    0xfffffffffffffffc\nrbx    0x5\nrcx    0x2eb5e2e\nrdx    0x80\nrdi    0x5\nrsi    0x7ffcb9d27d24\nrbp    0x7ffcb9d27cd8\nrsp    0x7ffcb9d27c90\nr8     0x0\nr9     0x0\nr10    0x393\nr11    0x246\nr12    0x7ffcb9d27d68\nr13    0x1\nr14    0xc9b9480\nr15    0x3\nrip    0x2eb5e2e\nrflags 0x246\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"33312dbc5fc68ebce09e9753f96df864fb727489a2e4bab2e61366b0d13c5f91","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"aed07da93803005b46e26e75e613559489562755fe1f89333bd605c689f5405d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"ed90acc1a208787b943d8fa885bcb7d65e2c641e8155aef811723d627a4dfb37","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"471f22e5967d4c6d260358df80d5f1c22cdca60ff8d89437ff00486c66a994f2","failure_pattern":"cluster has not yet reached Succeeded state","search_query":"cluster has not yet reached Succeeded state","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":7.142857142857143,"anchor_occurrences":2,"after_last_push_count":9,"trend_counts":[0,0,0,1,0,0,1,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":3,"prior_window_starts":["2026-06-08","2026-06-15","2026-06-22"],"prior_runs_affected":13,"prior_last_seen_at":"2026-06-25T23:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:199]: Timed out after 2520.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:197 with:\ncluster has not yet reached Succeeded state\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nto equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:199]: Timed out after 2520.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:197 with:\ncluster has not yet reached Succeeded state\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nto equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:199]: Timed out after 2520.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:197 with:\ncluster has not yet reached Succeeded state\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nto equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:199]: Timed out after 2520.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:197 with:\ncluster has not yet reached Succeeded state\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nto equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"ccee34282dc21302f7fa32a6cba8c9d0ba5e20e69efec394a8bdd9c07175ee39","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"7b311f6b09c3bb8e223e61a1567cbba9f44bc8fab49954456fd9328e616945db","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"79232f7fe5f1a65f3af30b7ba05ac2a1643a1a699ce682f3bc11178c03f0cbf4","failure_pattern":"failed to wait for first cluster \"\u003ccluster\u003e\" to complete creation (timeout '20.000000' minutes)","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":7.142857142857143,"anchor_occurrences":2,"after_last_push_count":7,"trend_counts":[0,0,0,1,0,0,1,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":2,"prior_window_starts":["2026-06-15","2026-06-22"],"prior_runs_affected":7,"prior_last_seen_at":"2026-06-23T05:14:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"fa82b432fd7a36953b4c71f37791bced4d6db1832370f07a5c77508bfa36ef7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","signature_id":"fa82b432fd7a36953b4c71f37791bced4d6db1832370f07a5c77508bfa36ef7e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"7137d3f115bf4c264bb74c888bdc6c3954a56a37032931fb9aa8d47ff31a4953","failure_pattern":"x509: certificate signed by unknown authority","search_query":"x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":7.142857142857143,"anchor_occurrences":2,"after_last_push_count":4,"trend_counts":[0,0,0,0,2,0,0,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-15"],"prior_runs_affected":4,"prior_last_seen_at":"2026-06-19T04:04:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:167]: Timed out after 600.001s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:167]: Timed out after 600.001s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:167]: Timed out after 600.003s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:167]: Timed out after 600.003s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2075354410581495808","occurred_at":"2026-07-09T23:00:24Z","signature_id":"55d4b1840d69212b2703b61209abbc219f27e5b689e1fef7a10a1d77779d4060","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075234996854984704","occurred_at":"2026-07-09T15:05:53Z","signature_id":"c95031350c87ed76ab8f37f77781a3df78d78ce53eee78d8acc4bb2657db707d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"b0e93a535dcc16a1c1adf87a06122b57bffeb9dc81f8d5d1e4ebca04cd2fa6a4","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v4.21.0-0.nightly-multi-2026-07-12-202543-nightly' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:736]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00075fe00\u003e: \n    failed to create HCP cluster np-dg-minor-n6g6vl: failed starting cluster creation \"np-dg-minor-n6g6vl\" in resourcegroup=\"rg-np-dg-minor-n6g6vl-nfnpp4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-n6g6vl-nfnpp4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-n6g6vl\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.21.0-0.nightly-multi-2026-07-12-202543-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:736]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00075fe00\u003e: \n    failed to create HCP cluster np-dg-minor-n6g6vl: failed starting cluster creation \"np-dg-minor-n6g6vl\" in resourcegroup=\"rg-np-dg-minor-n6g6vl-nfnpp4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-n6g6vl-nfnpp4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-n6g6vl\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.21.0-0.nightly-multi-2026-07-12-202543-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2076441601328025600","occurred_at":"2026-07-12T23:00:30Z","signature_id":"3b8bff5b360410492c0b39208bcc427c7fbaf2e93850e9c29f2dbbdb3e62888c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c06c4634c5f33bac64ee9a28a4387236f0ddb12cdeaa39ff33d99a21963b5e3f","failure_pattern":"VerifyFIPSEnabled failed: failed to check FIPS mode on node \u003cnode\u003e: failed to create FIPS check pod: pod \"\u003cpod\u003e\" is forbidden: error looking up service account \u003cnamespace\u003e/default: serviceaccount \"default\" not found","search_query":"VerifyFIPSEnabled failed: failed to check FIPS mode on node e9n0y8t3z6s7y7t-np-1-6l2x9-pkqwh: failed to create FIPS check pod: pods \"fips-check-\" is forbidden: error looking up service account e2e-fips-test-h4zck/default: serviceaccount \"default\" not found","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc000b267e0\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node e9n0y8t3z6s7y7t-np-1-6l2x9-pkqwh: failed to create FIPS check pod: pods \"fips-check-\" is forbidden: error looking up service account e2e-fips-test-h4zck/default: serviceaccount \"default\" not found\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc000b267e0\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node e9n0y8t3z6s7y7t-np-1-6l2x9-pkqwh: failed to create FIPS check pod: pods \"fips-check-\" is forbidden: error looking up service account e2e-fips-test-h4zck/default: serviceaccount \"default\" not found\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","signature_id":"240a061aeaa5d44d7837cb663804ed0d248d2a6adedea28dc723c04ef4e1326e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":1,"after_last_push_count":10,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":3,"prior_window_starts":["2026-06-08","2026-06-15","2026-06-22"],"prior_runs_affected":12,"prior_last_seen_at":"2026-06-22T04:30:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001a6a3a8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-rbkvwl, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-rbkvwl\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001a6a3a8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-rbkvwl, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-rbkvwl\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2075180010410348544","occurred_at":"2026-07-09T11:27:24Z","signature_id":"a27dc0b906c815e51578249fb35422c834e6dfab9313e1e3276046f6c522fafd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"79d555cc6e5bd17f4b96af0c9e2da6a571d8241aacf20f84304c98773ef2682b","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"np-ver-4-19-7\" for cluster \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-jbrspk\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-07-05..2026-07-12 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-15"],"prior_runs_affected":3,"prior_last_seen_at":"2026-06-19T21:23:09Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: failed to create node pool version 4.19.7 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f19d40\u003e: \n    failed waiting for nodepool=\"np-ver-4-19-7\" for cluster \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-jbrspk\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: failed to create node pool version 4.19.7 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f19d40\u003e: \n    failed waiting for nodepool=\"np-ver-4-19-7\" for cluster \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-jbrspk\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","signature_id":"42a89d117274e2a1b0c87982e6b4f88a7f679aa71d7faa4ff76fa840202f81fc","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}