{"meta":{"start_date":"2026-04-20","end_date":"2026-04-26","start_at":"2026-04-20T00:00:00Z","end_at":"2026-04-27T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T13:19:17Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":18,"failed_runs":17,"raw_occurrences":156,"matched_occurrences":156,"runs_affected":17},"rows":[{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mhh4lm, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mhh4lm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mhh4lm, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":99,"runs_affected":9,"failed_runs":9,"run_impact_percent":50,"anchor_occurrences":99,"after_last_push_count":99,"trend_counts":[0,0,1,2,6,23,35,32],"trend_range":"2026-04-19..2026-04-26 UTC","prior_windows_present":4,"prior_window_starts":["2026-03-23","2026-03-30","2026-04-06","2026-04-13"],"prior_runs_affected":34,"prior_last_seen_at":"2026-04-18T02:00:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00160e780\u003e: \n    failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-lv6fnh, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-lv6fnh\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-lv6fnh, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00160e780\u003e: \n    failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-lv6fnh, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-lv6fnh\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-lv6fnh, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009345d0\u003e: \n    failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-whhpxz, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-whhpxz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-whhpxz, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009345d0\u003e: \n    failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-whhpxz, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-whhpxz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-whhpxz, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000cbe3f0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-9rmffv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-9rmffv, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000cbe3f0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-9rmffv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-9rmffv, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"091379906f72b72c8716e255ca6a7ab5abeb5afe0a5154711f537ee6cccc7078","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"0abd5b447ee698b81f1707bd60ea9990ea759efcdcc972cf3200911f32951554","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"0c8eec4b448c989a4426a74b72aec2657e4d062e64c6bcd96691dd7a2d931c53","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"0ef18c5a73ad99cb3a2a7b35af140ba5b406753a710159114955666b9eca00ba","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"19376306f2cb2f8d3a34c840f2615729877d1101b37d66d169dab449359e4083","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"25641567eea06172ba6cb92bc7b2653b7a13b7ed8ec30d6bcb8671bfe48ab0eb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"337b3bcdf6bff71a570478cc431d07e080c480b6619d28bca13262e47613ca09","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"389f53eb08aaa262a4746d3d9df1c71749c4472ab81180e47886dd1718821289","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"399a40257d75cb5ffddcf870d190df6c60fe7d174ebdafc2970ef1225057d307","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"41b08b36509628977e0c4a29fd18cd907ffd9d873a916d8fc61ba0adc7ec6791","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"490e707732656bb205f0c2b7842ed396d133b31b4a489da5bef99790832fc47c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"4fce41b3b93955932fa55116f097b4af201d8d1bb7be911e1457047c023e9dd2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"6b842c3a6c5bf58fbc97686095bcbf0409ea27c7e6451d4f902c8f1e5041f639","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"7e150d909f159be9b5b28c44d39dacb5c4ae256b0edfd42c0e1b8f505520b27b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"7fabe0a0a77bc8b7d726cc32f55b76da2002a72646445f1a1380c793e15fa13f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"836f91f1cd8bfdcb0f96676459975cc2cb7c577ebb9e5319ec064f87ba28f637","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"878732d180af433b8995aec1e25accb54a0bd1f8571b38d54e9f2cab2e51f903","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"8b597f08f43ae682a1bdda5aad153f43e1c2aef983f36e3579154f9af1bb0201","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"93926f6c3672bf3976462b8daafe98f0cbc0830dd9baa34896cbedcf37cb7581","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"9bc6fd791bb6cbf6fab539a888a80559f0ce3d55ef74dc60926af2fe273e5834","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"b13db0749618eb55bb6b4ea4a674cd283003f154235f5b940a350eb0fbbaf2df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"ba662c0ce125a590e39c149172132dfe7ab95362572220bab74de87afa2b43ed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"bbf43f79c6a74e92b6455c17f090e462e7b43ce45f5c756585c890737ab9b260","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"c221e7013ea4fcac6bdf38d210d9546f99bedf0820ecea014803db00ce552805","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"d0f6120855a1b2832aceb656fedd514596f5d908f37c6256ec451b1b90c70e0d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"d5264602c365080c29189d40c4130bdc37966cee503cc99a25a11ce923a4428f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"e35e69fc71a500b0aea4b7c9fad71f2ca457959364405feab283d05411b76639","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"e451958423110a71d1ea4f2a18b5853a58b9708001e7b9aa92df7978027ca29a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"eb9783a4049e1fb83ac6d8bebfa5095eeeaebba84a9fa58a50e7ed1cbec762f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"ec50a6baede5053ee97065d9eeb3df2a36605205ba38b9ebfb42f3ac26c11e8c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"f84e9aa869e01832a7851d3aac2e566c7c9446050bb27e887945a04fad0709b2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"faa43504d0591365adc4ec1049ed25dc2a5ed93d482d2e94a7defc199479b1d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"01706e10b0337cbe655816d762082ee351a244e7a4679379050e4237fc118292","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"0c947ec904cce460e118b9fc35923f7adba2a08c5602b27e40798b6c8480f745","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"0cf775927920b903545ffcfb63bad531b2b3a7d2b877581c350b28de26fa5f09","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"0d2fe232df0c6cc18954a28c95125e289623c14c7d3d836ca7b4653f88d74144","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"14bbb864afc55c7528031b98a4d012877783e1ffed482a8497ddf6c2e899e3fb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"2cd74dbd07fad32a0115e5b5eb462a31724df47168b740700e7c8bcf47280b13","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"37f39781a008fbf5b3a41b3697cb620128399fc8e64355c052477bdf68e033e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"39e2a34db5789f9387e0f779c9561b05c8060410ee8034e46db6975beeb9eac5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"3a858e6463b7bfd31dc1b88aa682b44a075a64a2d706242b6b5a047df260ea07","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"44dd14381d2f7b5cc8011516dc7472dd159fdc4075b524b4cfd10f8e6653c4ac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"500552c80d5e6cf39edc1ac25ac4ec655f293aa720ca5ec6697fcda504eab180","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"53651d19cbe49cf67da131830453f8056aea73b272a3451b6bd0d6a2effd65d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"550e3c2907e6b86b314802113395ca01ff3056d51dfa05b53352cb3085c8d24f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"56f87dfc44bb2c177c58fd031b4fabb9980cd3ae51df1a11edaeaf618d4d3abd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"5eb94461da46babf1d327ea752a11c277193b482f7619928e1f689160d80e6bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"63e4b637a51b2c0da3807a562f501b3fa829e8086603b33a52f63ad351f07cbc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"720d3bceb28c5bcfade6285e9594227b3936787f7e99d0415193654d64626bd7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"764d59d57cbb27bae6783dff37766f5eee145beaa0443ab4309f7b75493aa69e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"7a5b998659431172e6af2cd7fd0c82f2cfdf04108ee82288f35f42b0d00662bd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"846208dc27ea83c438fa90b17bb06e350039ec279ae0ccd45ecf3f87ca8a4f25","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"891906deba439a676e1bc8a744a400ecbf84bea78809c11ac6dc74b3e77b268c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"89ba21b4bcfbbe6d107fb3a703667f597750ac9513ccaa6ad2f9d78b0d861da4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"9894fc32973dbc5594f0290a3b93313f5901ce4756d9dccccb7e1c3143690b43","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"9d1e0d7441116c3bc5b102726b67bfc06b6723a0ccd6dd584c5d05aaec695fc4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"9dfa414f9cb3b8aead60d1d8365f6e76c295769876c36b34e74c5915143729c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"b6347f263541480e1580363a58ffeb5a07ca84de4a3a5fdbc937e1634047c170","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"ca597d614d7ff2c6d515712d82f2982a5282a8950570929cedabd647eb099357","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"cc446d098103e3ce9d9a3e89c243910db33b829961b501ce80cf6a22c7fb6aa7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"db0ca79a61db809a7ac93518aac029323da413ecb3e25489916a7a2bd672e61c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"dcf73957f260152d1376565a8b721d4b4b8faf906f817cc857a47c61b5c4aa63","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"ea0510e42da51571b6a94e5c9584e4a2223a0df6e7b99a31b1ad23086d2e7016","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"eeaee2c8d18309cd7eae3899e842e4ddd07ec2d1aeb9b6e0fba08bbf65095ff2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"f1037a7e6423f9a41f2c6b0f7bbaf0e3d24d8cdbd7ff6cdfc820e63d9862baae","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"f5433af636af8234c3c981ec5ed789169c82dc05770c3361ae88467d5952b5ae","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"fd6b8e8029c177d26892c44f64f2626c46367821ab5870cfc0998c559e972d82","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"15370f7770de299dd21fa60d61b5548f9cade8895d265d537cc66db2f69caf2e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"1f7351eb1f3c9c713230b3d6aef5de456ff9923a7313aa4ce0582e85fb645aac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"31d893f41adc5c1cd26e71d1848f6a631e9135202d3c807638ec907e2acca29f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"67a15279bed7445a314209be249de1dd78609844878114eb517f01012354f7bd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"70dd085f4a557e8a91bf2c6b80f80b59a30dd84ecebc294e6ac8a55a2d45aeba","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"80558011f1c9e50f3e5aa79df8c7afbf4eb98cf5863b74baa0224d0ee6e5c781","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"83c702468523619fbe6379c16bc2a83ff8db23f70da8474d809616c96fb20f7a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"90e4e34401463acba6c3573ab41dffad0f272b7b03e5b5baa92d2cb894b15de9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"9beab1ecfa673ce13d1f36103123024a9be671eae0104f6d17b921a99c84f37f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"aa6d5a70477f4615dfac5aab96b8f8a7cbd905a70827fd3203948ca242c3fddc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"aabfb8cea29a0d179fe6ec205e918b3b7ae4ae354c1a30b9d4b895d189e11003","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"b06d6a76c2852dfffc3f3a52892611ed51c1ee0df7e6f5841a7a83fdefa28487","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"b3ce37421496a578a4b9a787cad5d606f4657435b3f432b26f60d042bc991f11","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"c3f3b3c04d0f9d55622e585be63d3577b725c1576b4458f258351592d55fc08f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"cb205895af951d198f72cc650fdb162c41a56581cf216a81b8b17d8ec1af2496","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"cf9d93b210380046fcc5f0bb98b4252e493abe88d50568fbb25ef6cdb25b4835","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"d2b32b3e6183f632a51d43179d4b9ccd70b9da37b667597cbc1014d0f19322f6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"d8adf81bb8ddcd49b4d64de9b008b395d4ac7042097b5453fa3f5cf731114b29","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047534526858530816","occurred_at":"2026-04-24T04:34:07Z","signature_id":"5ba612374b88287303367dd87819d840ddf45795ac363eda9583186c3d10f7a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047495744444698624","occurred_at":"2026-04-24T02:00:00Z","signature_id":"06f1523bb4cae24b536a5ddfd4e55a839d9f14750c796f75c0c08736e3ddb655","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047495744444698624","occurred_at":"2026-04-24T02:00:00Z","signature_id":"5890b5d4d3359e85a9cfea082995925e8cbe4f2930a0a58ae2ccd36d56f2f911","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047495744444698624","occurred_at":"2026-04-24T02:00:00Z","signature_id":"87a3c2a4d30f66aa511a30e792b6f5f5ab20e2ad4e07b5b342e1e298e5e7acfc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047495744444698624","occurred_at":"2026-04-24T02:00:00Z","signature_id":"c158c05d163e3bf5e103dada36b1956e1c81456aa6ddb91e4f05bff6d96bf201","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047105181333590016","occurred_at":"2026-04-23T00:08:03Z","signature_id":"3b38a593ad5e9f3ef36d5928f64a9da19e929bfb26b6cc5b8fa32c7782156a79","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047105181333590016","occurred_at":"2026-04-23T00:08:03Z","signature_id":"4765fac81880279dbb3c196a6b3dbc41f966dbe9b3d77391790f24ca99363ace","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047105181333590016","occurred_at":"2026-04-23T00:08:03Z","signature_id":"4bd2ef23f572ca3184bd95de7ab7614776b5c577a355de7621203178636245c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047105181333590016","occurred_at":"2026-04-23T00:08:03Z","signature_id":"8bfd559f19457b829e6cb216b527b9689411e98b3a294af6759a37cad1dcf1c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047105181333590016","occurred_at":"2026-04-23T00:08:03Z","signature_id":"ca3246957b750b911a874eb97e686489d8ea160b690c347586996411de708086","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047105181333590016","occurred_at":"2026-04-23T00:08:03Z","signature_id":"d9a36fe3f7dc5d9825e4ef35af6f2ecf4445891a636467cef49b6c627f4134bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046956383668015104","occurred_at":"2026-04-22T14:16:47Z","signature_id":"a020305e40755460769f708621143631e144b7929851ebafdb405f835006c286","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046914038591393792","occurred_at":"2026-04-22T11:28:31Z","signature_id":"333fe7ca0e7f018212d5389f30014334b1c60cbf991ac7284951a7c4115a1a44","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046408704101715968","occurred_at":"2026-04-21T02:00:30Z","signature_id":"30f48e83432edd63d3c985d4ccc7b95c2491f86f2d0671159cea664cbe675a9e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","failure_pattern":"tls: failed to verify certificate: x509: certificate is valid for \u003chcp-api-host\u003e, \u003chcp-api-host\u003e, not \u003chcp-api-host\u003e","search_query":"tls: failed to verify certificate: x509: certificate is valid for api.a3p3i1v4e6u2c5w.e9wf.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.a3p3i1v4e6u2c5w.v3e1.uksouth.aroapp-hcp.io","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":17,"runs_affected":6,"failed_runs":6,"run_impact_percent":33.333333333333336,"anchor_occurrences":17,"after_last_push_count":21,"trend_counts":[0,3,0,3,9,2,0,0],"trend_range":"2026-04-19..2026-04-26 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-23","2026-04-06","2026-04-13"],"prior_runs_affected":10,"prior_last_seen_at":"2026-04-15T09:25:00Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Unexpected error:\n    \u003c*errors.joinError | 0xc000796660\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.p0l1s7v5v6c3i0l.6i79.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.p0l1s7v5v6c3i0l.4dkw.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.p0l1s7v5v6c3i0l.6i79.uksouth.aroapp-hcp.io\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.p0l1s7v5v6c3i0l.6i79.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.p0l1s7v5v6c3i0l.4dkw.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.p0l1s7v5v6c3i0l.6i79.uksouth.aroapp-hcp.io\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Unexpected error:\n    \u003c*errors.joinError | 0xc000796660\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.p0l1s7v5v6c3i0l.6i79.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.p0l1s7v5v6c3i0l.4dkw.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.p0l1s7v5v6c3i0l.6i79.uksouth.aroapp-hcp.io\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.p0l1s7v5v6c3i0l.6i79.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.p0l1s7v5v6c3i0l.4dkw.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.p0l1s7v5v6c3i0l.6i79.uksouth.aroapp-hcp.io\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:124]: Unexpected error:\n    \u003c*errors.joinError | 0xc001156c00\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.n7a3d5c5g6v2q9o.vrfd.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.n7a3d5c5g6v2q9o.plmx.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.n7a3d5c5g6v2q9o.vrfd.uksouth.aroapp-hcp.io\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.n7a3d5c5g6v2q9o.vrfd.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.n7a3d5c5g6v2q9o.plmx.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.n7a3d5c5g6v2q9o.vrfd.uksouth.aroapp-hcp.io\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:124]: Unexpected error:\n    \u003c*errors.joinError | 0xc001156c00\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.n7a3d5c5g6v2q9o.vrfd.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.n7a3d5c5g6v2q9o.plmx.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.n7a3d5c5g6v2q9o.vrfd.uksouth.aroapp-hcp.io\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.n7a3d5c5g6v2q9o.vrfd.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.n7a3d5c5g6v2q9o.plmx.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.n7a3d5c5g6v2q9o.vrfd.uksouth.aroapp-hcp.io\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:161]: Unexpected error:\n    \u003c*url.Error | 0xc00174e600\u003e: \n    Get \"https://api.t9i0s9f7m4g0f5e.0122.uksouth.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": tls: failed to verify certificate: x509: certificate is valid for api.t9i0s9f7m4g0f5e.rzw9.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.t9i0s9f7m4g0f5e.0122.uksouth.aroapp-hcp.io\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:161]: Unexpected error:\n    \u003c*url.Error | 0xc00174e600\u003e: \n    Get \"https://api.t9i0s9f7m4g0f5e.0122.uksouth.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": tls: failed to verify certificate: x509: certificate is valid for api.t9i0s9f7m4g0f5e.rzw9.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.t9i0s9f7m4g0f5e.0122.uksouth.aroapp-hcp.io\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047495744444698624","occurred_at":"2026-04-24T02:00:00Z","signature_id":"041e08e0f69fcee44ad56e8a175df8fa789ebf99065041189b005bda51546fdb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047495744444698624","occurred_at":"2026-04-24T02:00:00Z","signature_id":"23cc9dae185dfede24507bc59941bd312c14aa6e0d3da7482a07abc571f45daf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047452573052440576","occurred_at":"2026-04-23T23:08:27Z","signature_id":"2364c917ae0cf12ed69fbb25e4c0413548f8ec6407144980cf6b33fbdae36ac4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047452573052440576","occurred_at":"2026-04-23T23:08:27Z","signature_id":"7854c7962454d39e05f1f3f3d9816cb4edb55d46ae034464371911e60b844295","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047452573052440576","occurred_at":"2026-04-23T23:08:27Z","signature_id":"78e2630b9333b13334c561e7f2f476be7dfbb53e9888cce591f72bdb67b90d20","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047452573052440576","occurred_at":"2026-04-23T23:08:27Z","signature_id":"7e06bea11cbb64d5a45c82a4c15051b8638201463ad777ef0ac3d9f5a30d3672","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047452573052440576","occurred_at":"2026-04-23T23:08:27Z","signature_id":"8128997d224e82a57c3c7748dd9d74ef03e04eea3cb9a5861a74bf5ab4902946","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047452573052440576","occurred_at":"2026-04-23T23:08:27Z","signature_id":"c0883f339955db3f2f7834e4ee94e66a2fede7e2aaca213690a34560424401af","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047328993031491584","occurred_at":"2026-04-23T14:57:24Z","signature_id":"18e48067dd9948d3342a0d59d3cc16a366a1d457a85af56e2e788cd9a1e689a9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047328993031491584","occurred_at":"2026-04-23T14:57:24Z","signature_id":"c1da7bef0bb2e4746842f7e901cf5beb839a2a61345ef0c4619c637b5f1bd559","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047328993031491584","occurred_at":"2026-04-23T14:57:24Z","signature_id":"e8dd1822829579cc99aebe864a873cc56f69bff8d1e602795abe27828cd5b6ef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047024789423394816","occurred_at":"2026-04-22T18:48:36Z","signature_id":"6736c55513db9c62b4e7b1d4817a91d50f048565ffe1d4242804bd6d2d498e88","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047024789423394816","occurred_at":"2026-04-22T18:48:36Z","signature_id":"7fe72b9b19f5e7fead9f3de4a2cd748b499d2f808a00b615669fe1eafe479696","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046879318772027392","occurred_at":"2026-04-22T09:10:33Z","signature_id":"17fa8a755d5fbe5fb9413c3d9a98023fd7c2539d192561972224f6cbee3c6d7c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046046207125491712","occurred_at":"2026-04-20T02:00:04Z","signature_id":"1005de253f4229d1b1e34e9d833e0b18059f7431a8b94dc7c31d84a49a268ed5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046046207125491712","occurred_at":"2026-04-20T02:00:04Z","signature_id":"983604bad68098fb0a45fb07045171bcd0da177b95925c426788e8c8ecded3bc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046046207125491712","occurred_at":"2026-04-20T02:00:04Z","signature_id":"da3f7f58d6c27dfd98f711bd046985429377570443c58fba1b1ed57bb45506a4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f7692883b7d8edf2d0661368bcbf0596b25bb01a118dcf71cc62820145611635","failure_pattern":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set: assertion failed: expected values to equal","search_query":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set: assertion failed: expected values to equal","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":38.888888888888886,"anchor_occurrences":7,"after_last_push_count":7,"trend_counts":[0,0,0,0,3,3,1,0],"trend_range":"2026-04-19..2026-04-26 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set","occurrences":7}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/mise_routing.go:89]: Expected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/mise_routing.go:89]: Expected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047534526858530816","occurred_at":"2026-04-24T04:34:07Z","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047495744444698624","occurred_at":"2026-04-24T02:00:00Z","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047452573052440576","occurred_at":"2026-04-23T23:08:27Z","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047328993031491584","occurred_at":"2026-04-23T14:57:24Z","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047231067995246592","occurred_at":"2026-04-23T08:28:16Z","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":33.333333333333336,"anchor_occurrences":6,"after_last_push_count":6,"trend_counts":[0,0,0,4,2,0,0,0],"trend_range":"2026-04-19..2026-04-26 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":6}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: Unexpected error:\n    \u003c*errors.joinError | 0xc000060d20\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-wt4d9m\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: Unexpected error:\n    \u003c*errors.joinError | 0xc000060d20\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-wt4d9m\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: Unexpected error:\n    \u003c*errors.joinError | 0xc001242648\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-qkvnrj\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: Unexpected error:\n    \u003c*errors.joinError | 0xc001242648\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-qkvnrj\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e508b8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-fjrggl\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e508b8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-fjrggl\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047231067995246592","occurred_at":"2026-04-23T08:28:16Z","signature_id":"c8cfb15f0633bd15ae0ff18324df4f7cd8aae4df9cd2da599a5448c61107a212","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047105181333590016","occurred_at":"2026-04-23T00:08:03Z","signature_id":"0be642cb7b1b860a56a9c13ab1eac783e14b5f53c75ccf658ee52fd8c0807f6e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047024789423394816","occurred_at":"2026-04-22T18:48:36Z","signature_id":"48159465318ae7cb16fd7133a8f7cdcf03097f41d54cb30cea70261ed9054c8f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046956383668015104","occurred_at":"2026-04-22T14:16:47Z","signature_id":"b03d1fadc884dc16e19a954a64e384b01f7091cef8db05565dd9a7755a124e12","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046914038591393792","occurred_at":"2026-04-22T11:28:31Z","signature_id":"7f1df1cc9a46d12be1958e532b4c5752baeb878aacfd79d0bcd706518d43d67e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046879318772027392","occurred_at":"2026-04-22T09:10:33Z","signature_id":"f80b7ea1525f797da1e97c41f8eb510f1337f4e1acde91deda06ef14f08873dc","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":5,"runs_affected":2,"failed_runs":2,"run_impact_percent":11.11111111111111,"anchor_occurrences":5,"after_last_push_count":93,"trend_counts":[0,0,0,0,0,3,0,2],"trend_range":"2026-04-19..2026-04-26 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-23","2026-03-30","2026-04-06"],"prior_runs_affected":28,"prior_last_seen_at":"2026-04-10T10:59:58Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1}],"full_error_samples":["onsi/ginkgo/v2.generateTable.func2.1({0xa0ccd80?, 0xc0009569c0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:293 +0x105 fp=0xc000b91f88 sp=0xc000b91f18 pc=0x3978245\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc000b91fe0 sp=0xc000b91f88 pc=0x3969bbb\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000b91fe8 sp=0xc000b91fe0 pc=0x34facc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 83\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\ngoroutine 179 gp=0xc000fa6fc0 m=nil [IO wait]:\nruntime.gopark(0xc000d914c0?, 0x44ab00e8?, 0x0?, 0x42?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000ed9720 sp=0xc000ed9700 pc=0x34f250e\nruntime.netpollblock(0x355cd18?, 0x347e066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc000ed9758 sp=0xc000ed9720 pc=0x34b1fb7\ninternal/poll.runtime_pollWait(0x7fd445b7d200, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc000ed9778 sp=0xc000ed9758 pc=0x34f0da5\ninternal/poll.(*pollDesc).wait(0xc00050fe00?, 0xc0012ac000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc000ed97a0 sp=0xc000ed9778 pc=0x357fa47\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc00050fe00, {0xc0012ac000, 0x3500, 0x3500})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc000ed9838 sp=0xc000ed97a0 pc=0x3580e79\nnet.(*netFD).Read(0xc00050fe00, {0xc0012ac000?, 0xc0012ac2cb?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc000ed9880 sp=0xc000ed9838 pc=0x37465e5\nnet.(*conn).Read(0xc0001a4a60, {0xc0012ac000?, 0x7fd43cfc2680?, 0x7fd48d7a55c0?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc000ed98c8 sp=0xc000ed9880 pc=0x3758685\ncrypto/tls.(*atLeastReader).Read(0xc00100a1e0, {0xc0012ac000?, 0x3230?, 0x34ef245?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc000ed9910 sp=0xc000ed98c8 pc=0x37c411b\nbytes.(*Buffer).ReadFrom(0xc000d1c628, {0x9feefe0, 0xc00100a1e0})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc000ed9968 sp=0xc000ed9910 pc=0x35bdc78\ncrypto/tls.(*Conn).readFromUntil(0xc000d1c388, {0x9fefb80, 0xc0001a4a60}, 0xc000ed99d0?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc000ed99a0 sp=0xc000ed9968 pc=0x37c42fe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc000d1c388, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc000ed9c18 sp=0xc000ed99a0 pc=0x37c137b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc000d1c388, {0xc001113000, 0x1000, 0x3bc0820?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc000ed9c80 sp=0xc000ed9c18 pc=0x37c7c65\nbufio.(*Reader).Read(0xc00061c9c0, {0xc000eea664, 0x9, 0x3bce8ae?})\n\t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0xc000ed9cb8 sp=0xc000ed9c80 pc=0x365a1d7\nio.ReadAtLeast({0x9fee5e0, 0xc00061c9c0}, {0xc000eea664, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0xc000ed9d00 sp=0xc000ed9cb8 pc=0x354ce8e\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc000eea664, 0x9, 0xc000000067?}, {0x9fee5e0?, 0xc00061c9c0?})\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:250 +0x65 fp=0xc000ed9d50 sp=0xc000ed9d00 pc=0x3bb75e5\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc000eea620)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:513 +0x6b fp=0xc000ed9dd8 sp=0xc000ed9d50 pc=0x3bb81ab\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc000eea620)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:572 +0x18 fp=0xc000ed9e18 sp=0xc000ed9dd8 pc=0x3bb8918\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000ed9fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2251 +0xca fp=0xc000ed9f60 sp=0xc000ed9e18 pc=0x3bcb7ea\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc000fa7500)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2120 +0x52 fp=0xc000ed9fc8 sp=0xc000ed9f60 pc=0x3bcab72\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0x25 fp=0xc000ed9fe0 sp=0xc000ed9fc8 pc=0x3bc4305\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000ed9fe8 sp=0xc000ed9fe0 pc=0x34facc1\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 178\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0xda5\n\nrax    0xca\nrbx    0x0\nrcx    0x34fcae3\nrdx    0x0\nrdi    0xdb9fcb0\nrsi    0x80\nrbp    0x7fff160bed10\nrsp    0x7fff160becc8\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x12\nr13    0x1\nr14    0xdb9dbc0\nr15    0x5\nrip    0x34fcae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command\nonsi/ginkgo/v2.generateTable.func2.1({0xa0ccd80?, 0xc0009569c0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:293 +0x105 fp=0xc000b91f88 sp=0xc000b91f18 pc=0x3978245\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc000b91fe0 sp=0xc000b91f88 pc=0x3969bbb\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000b91fe8 sp=0xc000b91fe0 pc=0x34facc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 83\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\ngoroutine 179 gp=0xc000fa6fc0 m=nil [IO wait]:\nruntime.gopark(0xc000d914c0?, 0x44ab00e8?, 0x0?, 0x42?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000ed9720 sp=0xc000ed9700 pc=0x34f250e\nruntime.netpollblock(0x355cd18?, 0x347e066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc000ed9758 sp=0xc000ed9720 pc=0x34b1fb7\ninternal/poll.runtime_pollWait(0x7fd445b7d200, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc000ed9778 sp=0xc000ed9758 pc=0x34f0da5\ninternal/poll.(*pollDesc).wait(0xc00050fe00?, 0xc0012ac000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc000ed97a0 sp=0xc000ed9778 pc=0x357fa47\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc00050fe00, {0xc0012ac000, 0x3500, 0x3500})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc000ed9838 sp=0xc000ed97a0 pc=0x3580e79\nnet.(*netFD).Read(0xc00050fe00, {0xc0012ac000?, 0xc0012ac2cb?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc000ed9880 sp=0xc000ed9838 pc=0x37465e5\nnet.(*conn).Read(0xc0001a4a60, {0xc0012ac000?, 0x7fd43cfc2680?, 0x7fd48d7a55c0?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc000ed98c8 sp=0xc000ed9880 pc=0x3758685\ncrypto/tls.(*atLeastReader).Read(0xc00100a1e0, {0xc0012ac000?, 0x3230?, 0x34ef245?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc000ed9910 sp=0xc000ed98c8 pc=0x37c411b\nbytes.(*Buffer).ReadFrom(0xc000d1c628, {0x9feefe0, 0xc00100a1e0})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc000ed9968 sp=0xc000ed9910 pc=0x35bdc78\ncrypto/tls.(*Conn).readFromUntil(0xc000d1c388, {0x9fefb80, 0xc0001a4a60}, 0xc000ed99d0?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc000ed99a0 sp=0xc000ed9968 pc=0x37c42fe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc000d1c388, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc000ed9c18 sp=0xc000ed99a0 pc=0x37c137b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc000d1c388, {0xc001113000, 0x1000, 0x3bc0820?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc000ed9c80 sp=0xc000ed9c18 pc=0x37c7c65\nbufio.(*Reader).Read(0xc00061c9c0, {0xc000eea664, 0x9, 0x3bce8ae?})\n\t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0xc000ed9cb8 sp=0xc000ed9c80 pc=0x365a1d7\nio.ReadAtLeast({0x9fee5e0, 0xc00061c9c0}, {0xc000eea664, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0xc000ed9d00 sp=0xc000ed9cb8 pc=0x354ce8e\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc000eea664, 0x9, 0xc000000067?}, {0x9fee5e0?, 0xc00061c9c0?})\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:250 +0x65 fp=0xc000ed9d50 sp=0xc000ed9d00 pc=0x3bb75e5\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc000eea620)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:513 +0x6b fp=0xc000ed9dd8 sp=0xc000ed9d50 pc=0x3bb81ab\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc000eea620)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:572 +0x18 fp=0xc000ed9e18 sp=0xc000ed9dd8 pc=0x3bb8918\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000ed9fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2251 +0xca fp=0xc000ed9f60 sp=0xc000ed9e18 pc=0x3bcb7ea\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc000fa7500)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2120 +0x52 fp=0xc000ed9fc8 sp=0xc000ed9f60 pc=0x3bcab72\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0x25 fp=0xc000ed9fe0 sp=0xc000ed9fc8 pc=0x3bc4305\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000ed9fe8 sp=0xc000ed9fe0 pc=0x34facc1\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 178\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0xda5\n\nrax    0xca\nrbx    0x0\nrcx    0x34fcae3\nrdx    0x0\nrdi    0xdb9fcb0\nrsi    0x80\nrbp    0x7fff160bed10\nrsp    0x7fff160becc8\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x12\nr13    0x1\nr14    0xdb9dbc0\nr15    0x5\nrip    0x34fcae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command","/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:199\n          \u003e github.com/onsi/ginkgo/v2/internal.(*group).run(0xc000e24b88, {0xc000b26620?, 0x0?, 0x0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:349\n          \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).runSpecs(0xc000a4aa88, {0x0, 0x0}, {0xdbcaa00, 0x0, 0x0}, {0xc0000be094, 0x2a}, 0x0, {0xc000b26600, ...})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:496\n          \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).RunSpec(_, {_, _}, {_, _, _}, {_, _}, {0xc0000be094, 0x2a}, ...)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite_patch.go:68\n            github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite.func1.1({0x3481a85?, 0xc000b25980?})\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/ginkgo/util.go:85\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.runSpec({0xa0b0da8, 0xc00060dcc0}, 0xc000d7e000, 0x1)\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:596\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:237\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:230\n\n          goroutine 53 [select]\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2(0xc000d7e000?)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:131\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts in goroutine 94\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:128\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:115 @ 04/26/26 14:20:43.171\n  \"ts\"=\"2026-04-26 14:20:43.172175\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-26 14:20:43.172316\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-candidate-4-21-w8wp9g-l44gtn\" \"resourceGroup\"=\"rg-candidate-4-21-nl8wck\"\n  \"ts\"=\"2026-04-26 14:20:43.370310\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-candidate-4-21-w8wp9g-l44gtn\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-26 14:20:44.473602\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-6427b6424c4038306dc527fd5f7fc0b377fe9c26e56432adfe480dff6189e0c7.yaml.gz\"\n  \"ts\"=\"2026-04-26 14:20:44.473758\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-6427b6424c4038306dc527fd5f7fc0b377fe9c26e56432adfe480dff6189e0c7.yaml.gz\"\n  \"ts\"=\"2026-04-26 14:20:44.473903\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-26 14:20:44.476357\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-117\"\n  \"ts\"=\"2026-04-26 14:20:44.476386\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-candidate-4-21-nl8wck\"\n  \"ts\"=\"2026-04-26 14:20:44.634596\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-117\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-26 14:20:44.648345\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-candidate-4-21-nl8wck\" \"deployment\"=\"pooledNonMsiScopedAssignments-hdyao36g42hfg\"\n  \"ts\"=\"2026-04-26 14:20:44.765715\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-117\" \"deployment\"=\"msi-deployment-117\"\n  \"ts\"=\"2026-04-26 14:20:44.786703\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-candidate-4-21-nl8wck\" \"deployment\"=\"customer-infra-cluster-candidate-4-21-w8wp9g-l44gtn\"\n  \"ts\"=\"2026-04-26 14:20:45.009501\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-26 14:20:45.327005\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-26 14:20:45.327061\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-candidate-4-21-nl8wck\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:199\n          \u003e github.com/onsi/ginkgo/v2/internal.(*group).run(0xc000e24b88, {0xc000b26620?, 0x0?, 0x0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:349\n          \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).runSpecs(0xc000a4aa88, {0x0, 0x0}, {0xdbcaa00, 0x0, 0x0}, {0xc0000be094, 0x2a}, 0x0, {0xc000b26600, ...})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:496\n          \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).RunSpec(_, {_, _}, {_, _, _}, {_, _}, {0xc0000be094, 0x2a}, ...)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite_patch.go:68\n            github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite.func1.1({0x3481a85?, 0xc000b25980?})\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/ginkgo/util.go:85\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.runSpec({0xa0b0da8, 0xc00060dcc0}, 0xc000d7e000, 0x1)\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:596\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:237\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:230\n\n          goroutine 53 [select]\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2(0xc000d7e000?)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:131\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts in goroutine 94\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:128\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:115 @ 04/26/26 14:20:43.171\n  \"ts\"=\"2026-04-26 14:20:43.172175\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-26 14:20:43.172316\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-candidate-4-21-w8wp9g-l44gtn\" \"resourceGroup\"=\"rg-candidate-4-21-nl8wck\"\n  \"ts\"=\"2026-04-26 14:20:43.370310\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-candidate-4-21-w8wp9g-l44gtn\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-26 14:20:44.473602\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-6427b6424c4038306dc527fd5f7fc0b377fe9c26e56432adfe480dff6189e0c7.yaml.gz\"\n  \"ts\"=\"2026-04-26 14:20:44.473758\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-6427b6424c4038306dc527fd5f7fc0b377fe9c26e56432adfe480dff6189e0c7.yaml.gz\"\n  \"ts\"=\"2026-04-26 14:20:44.473903\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-26 14:20:44.476357\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-117\"\n  \"ts\"=\"2026-04-26 14:20:44.476386\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-candidate-4-21-nl8wck\"\n  \"ts\"=\"2026-04-26 14:20:44.634596\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-117\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-26 14:20:44.648345\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-candidate-4-21-nl8wck\" \"deployment\"=\"pooledNonMsiScopedAssignments-hdyao36g42hfg\"\n  \"ts\"=\"2026-04-26 14:20:44.765715\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-117\" \"deployment\"=\"msi-deployment-117\"\n  \"ts\"=\"2026-04-26 14:20:44.786703\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-candidate-4-21-nl8wck\" \"deployment\"=\"customer-infra-cluster-candidate-4-21-w8wp9g-l44gtn\"\n  \"ts\"=\"2026-04-26 14:20:45.009501\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-26 14:20:45.327005\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-26 14:20:45.327061\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-candidate-4-21-nl8wck\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","b78 pc=0x7addf0e\ngithub.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:237 +0x196 fp=0xc000e97fe0 sp=0xc000e97c20 pc=0x7ad9c76\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000e97fe8 sp=0xc000e97fe0 pc=0x34fbcc1\ncreated by github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:230 +0x1e5\n\ngoroutine 154 gp=0xc0006036c0 m=nil [IO wait]:\nruntime.gopark(0xc000c94a20?, 0x84f4a900?, 0x0?, 0xd6?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000e1d720 sp=0xc000e1d700 pc=0x34f350e\nruntime.netpollblock(0x355dd18?, 0x347f066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc000e1d758 sp=0xc000e1d720 pc=0x34b2fb7\ninternal/poll.runtime_pollWait(0x7fd885f9b000, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc000e1d778 sp=0xc000e1d758 pc=0x34f1da5\ninternal/poll.(*pollDesc).wait(0xc0005dcc80?, 0xc000e24000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc000e1d7a0 sp=0xc000e1d778 pc=0x3580a47\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc0005dcc80, {0xc000e24000, 0x3500, 0x3500})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc000e1d838 sp=0xc000e1d7a0 pc=0x3581e79\nnet.(*netFD).Read(0xc0005dcc80, {0xc000e24000?, 0xc000e242f2?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc000e1d880 sp=0xc000e1d838 pc=0x37475e5\nnet.(*conn).Read(0xc000d96418, {0xc000e24000?, 0x7fd8844dfd00?, 0x7fd8cdbc3a78?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc000e1d8c8 sp=0xc000e1d880 pc=0x3759685\ncrypto/tls.(*atLeastReader).Read(0xc000ada018, {0xc000e24000?, 0x3209?, 0x34f0245?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc000e1d910 sp=0xc000e1d8c8 pc=0x37c511b\nbytes.(*Buffer).ReadFrom(0xc000c717a8, {0x9ff3c00, 0xc000ada018})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc000e1d968 sp=0xc000e1d910 pc=0x35bec78\ncrypto/tls.(*Conn).readFromUntil(0xc000c71508, {0x9ff47a0, 0xc000d96418}, 0xc000e1d9d0?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc000e1d9a0 sp=0xc000e1d968 pc=0x37c52fe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc000c71508, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc000e1dc18 sp=0xc000e1d9a0 pc=0x37c237b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc000c71508, {0xc00131d000, 0x1000, 0x3bc1820?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc000e1dc80 sp=0xc000e1dc18 pc=0x37c8c65\nbufio.(*Reader).Read(0xc0012fa6c0, {0xc000156c84, 0x9, 0x3bcf8ae?})\n\t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0xc000e1dcb8 sp=0xc000e1dc80 pc=0x365b1d7\nio.ReadAtLeast({0x9ff3200, 0xc0012fa6c0}, {0xc000156c84, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0xc000e1dd00 sp=0xc000e1dcb8 pc=0x354de8e\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc000156c84, 0x9, 0xc000000211?}, {0x9ff3200?, 0xc0012fa6c0?})\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:250 +0x65 fp=0xc000e1dd50 sp=0xc000e1dd00 pc=0x3bb85e5\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc000156c40)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:513 +0x6b fp=0xc000e1ddd8 sp=0xc000e1dd50 pc=0x3bb91ab\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc000156c40)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:572 +0x18 fp=0xc000e1de18 sp=0xc000e1ddd8 pc=0x3bb9918\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000e1dfa8)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2251 +0xca fp=0xc000e1df60 sp=0xc000e1de18 pc=0x3bcc7ea\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc00140a8c0)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2120 +0x52 fp=0xc000e1dfc8 sp=0xc000e1df60 pc=0x3bcbb72\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0x25 fp=0xc000e1dfe0 sp=0xc000e1dfc8 pc=0x3bc5305\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000e1dfe8 sp=0xc000e1dfe0 pc=0x34fbcc1\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 153\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0xda5\n\nrax    0xfffffffffffffffc\nrbx    0x0\nrcx    0x34fdae3\nrdx    0x0\nrdi    0xdba8e10\nrsi    0x80\nrbp    0x7fd886fcfcb8\nrsp    0x7fd886fcfc70\nr8     0x0\nr9     0x0\nr10    0x7fd886fcfca8\nr11    0x202\nr12    0x7fd886fcfca8\nr13    0x1\nr14    0xc000002c40\nr15    0x0\nrip    0x34fdae3\nrflags 0x202\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command\nb78 pc=0x7addf0e\ngithub.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:237 +0x196 fp=0xc000e97fe0 sp=0xc000e97c20 pc=0x7ad9c76\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000e97fe8 sp=0xc000e97fe0 pc=0x34fbcc1\ncreated by github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:230 +0x1e5\n\ngoroutine 154 gp=0xc0006036c0 m=nil [IO wait]:\nruntime.gopark(0xc000c94a20?, 0x84f4a900?, 0x0?, 0xd6?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000e1d720 sp=0xc000e1d700 pc=0x34f350e\nruntime.netpollblock(0x355dd18?, 0x347f066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc000e1d758 sp=0xc000e1d720 pc=0x34b2fb7\ninternal/poll.runtime_pollWait(0x7fd885f9b000, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc000e1d778 sp=0xc000e1d758 pc=0x34f1da5\ninternal/poll.(*pollDesc).wait(0xc0005dcc80?, 0xc000e24000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc000e1d7a0 sp=0xc000e1d778 pc=0x3580a47\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc0005dcc80, {0xc000e24000, 0x3500, 0x3500})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc000e1d838 sp=0xc000e1d7a0 pc=0x3581e79\nnet.(*netFD).Read(0xc0005dcc80, {0xc000e24000?, 0xc000e242f2?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc000e1d880 sp=0xc000e1d838 pc=0x37475e5\nnet.(*conn).Read(0xc000d96418, {0xc000e24000?, 0x7fd8844dfd00?, 0x7fd8cdbc3a78?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc000e1d8c8 sp=0xc000e1d880 pc=0x3759685\ncrypto/tls.(*atLeastReader).Read(0xc000ada018, {0xc000e24000?, 0x3209?, 0x34f0245?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc000e1d910 sp=0xc000e1d8c8 pc=0x37c511b\nbytes.(*Buffer).ReadFrom(0xc000c717a8, {0x9ff3c00, 0xc000ada018})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc000e1d968 sp=0xc000e1d910 pc=0x35bec78\ncrypto/tls.(*Conn).readFromUntil(0xc000c71508, {0x9ff47a0, 0xc000d96418}, 0xc000e1d9d0?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc000e1d9a0 sp=0xc000e1d968 pc=0x37c52fe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc000c71508, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc000e1dc18 sp=0xc000e1d9a0 pc=0x37c237b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc000c71508, {0xc00131d000, 0x1000, 0x3bc1820?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc000e1dc80 sp=0xc000e1dc18 pc=0x37c8c65\nbufio.(*Reader).Read(0xc0012fa6c0, {0xc000156c84, 0x9, 0x3bcf8ae?})\n\t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0xc000e1dcb8 sp=0xc000e1dc80 pc=0x365b1d7\nio.ReadAtLeast({0x9ff3200, 0xc0012fa6c0}, {0xc000156c84, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0xc000e1dd00 sp=0xc000e1dcb8 pc=0x354de8e\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc000156c84, 0x9, 0xc000000211?}, {0x9ff3200?, 0xc0012fa6c0?})\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:250 +0x65 fp=0xc000e1dd50 sp=0xc000e1dd00 pc=0x3bb85e5\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc000156c40)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:513 +0x6b fp=0xc000e1ddd8 sp=0xc000e1dd50 pc=0x3bb91ab\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc000156c40)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:572 +0x18 fp=0xc000e1de18 sp=0xc000e1ddd8 pc=0x3bb9918\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000e1dfa8)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2251 +0xca fp=0xc000e1df60 sp=0xc000e1de18 pc=0x3bcc7ea\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc00140a8c0)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2120 +0x52 fp=0xc000e1dfc8 sp=0xc000e1df60 pc=0x3bcbb72\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0x25 fp=0xc000e1dfe0 sp=0xc000e1dfc8 pc=0x3bc5305\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000e1dfe8 sp=0xc000e1dfe0 pc=0x34fbcc1\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 153\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0xda5\n\nrax    0xfffffffffffffffc\nrbx    0x0\nrcx    0x34fdae3\nrdx    0x0\nrdi    0xdba8e10\nrsi    0x80\nrbp    0x7fd886fcfcb8\nrsp    0x7fd886fcfc70\nr8     0x0\nr9     0x0\nr10    0x7fd886fcfca8\nr11    0x202\nr12    0x7fd886fcfca8\nr13    0x1\nr14    0xc000002c40\nr15    0x0\nrip    0x34fdae3\nrflags 0x202\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"135a8d090682264ff405d4580f350f5b62f5f9deea9dbb6faf885ee973d302b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"e2b4b5977905781ce553eb3fff22f33d157c0295fcd95cc56c9f33a8695c46dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"8cecea8915e316fe9076db2275bc1d6b45f4ade5b0c59dcb9235d6dbd7f98c1d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"dcaaecc2f5339a1c97f554317e0c4140b84b4db3579805e70f8a471d03dc8303","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"f16cff6c452324b1a7add76b103260b56495045e2dca2aed19839af0d7d5eebc","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e9c6c185c09aa6c23477c53c1d3d7fb1bb60e041c3cb718558ce145b6718d4d","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":22.22222222222222,"anchor_occurrences":4,"after_last_push_count":6,"trend_counts":[0,0,0,0,1,1,1,1],"trend_range":"2026-04-19..2026-04-26 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-30","2026-04-06"],"prior_runs_affected":6,"prior_last_seen_at":"2026-04-10T02:06:29Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.000s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.000s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2707.498s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2707.498s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2713.441s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2713.441s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"8a441c3cc9a1724fd565ecc68f397cdfdb078c056f41b0ab5661bd4e719eb328","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"4f31d7d247ea42e46469d3f77ee2c032edd582aa8c4c0f831b3f08a436aeb817","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047534526858530816","occurred_at":"2026-04-24T04:34:07Z","signature_id":"8f5944fc31ef82fc3165f013626b18f0d15de769e768b4f09402c923662fe429","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047105181333590016","occurred_at":"2026-04-23T00:08:03Z","signature_id":"0c4ed9bba8fe112ea7971d2241bf4fe4d2d0df6edae20700d459cff3d63baf26","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"9626adf85231cae7571909c00084bdfb7c9de3df3208a3d5200ee95baf8fc7e8","failure_pattern":"context deadline exceeded","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":16.666666666666668,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,1,0,1,1],"trend_range":"2026-04-19..2026-04-26 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: Unexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: Unexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"b15c55d32190fb3d32f16f58a52f080055a52cddf394a6cd7e062c8467f788e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047843062960361472","occurred_at":"2026-04-25T01:00:03Z","signature_id":"b15c55d32190fb3d32f16f58a52f080055a52cddf394a6cd7e062c8467f788e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047105181333590016","occurred_at":"2026-04-23T00:08:03Z","signature_id":"b15c55d32190fb3d32f16f58a52f080055a52cddf394a6cd7e062c8467f788e4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-kpr8f5, error: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-kpr8f5\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":11.11111111111111,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,1,2,0,0],"trend_range":"2026-04-19..2026-04-26 UTC","prior_windows_present":2,"prior_window_starts":["2026-04-06","2026-04-13"],"prior_runs_affected":8,"prior_last_seen_at":"2026-04-15T06:04:28Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:242]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009d9500\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-cvqx4g, error: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-cvqx4g\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:242]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009d9500\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-cvqx4g, error: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-cvqx4g\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:94]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016815f0\u003e: \n    failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-kpr8f5, error: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-kpr8f5\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:94]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016815f0\u003e: \n    failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-kpr8f5, error: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-kpr8f5\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008ec4b0\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-htjcmf, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-htjcmf\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008ec4b0\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-htjcmf, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-htjcmf\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"81a038536c4312c97a10ccb7cbcc503a577979e412955ce6733518b54b4004d0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"9a53c71edfc949be9d2dc81bb8243a9e3703f4e1ab90c33a165336c9cff54109","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047328993031491584","occurred_at":"2026-04-23T14:57:24Z","signature_id":"9bef192e78ada1b3ccf602133b1cd3333a57f42a2080d6ca2240cf5ca3bffd8a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":11.11111111111111,"anchor_occurrences":2,"after_last_push_count":39,"trend_counts":[0,0,0,1,0,1,0,0],"trend_range":"2026-04-19..2026-04-26 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-30","2026-04-06","2026-04-13"],"prior_runs_affected":19,"prior_last_seen_at":"2026-04-16T02:00:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000ae7da0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-k6db5s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/195989b9-9d82-430b-b464-5e8928304bdb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/195989b9-9d82-430b-b464-5e8928304bdb\",\n      \"name\": \"195989b9-9d82-430b-b464-5e8928304bdb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-04-24T05:01:33.488341598Z\",\n      \"endTime\": \"2026-04-24T05:02:10.881244554Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000ae7da0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-k6db5s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/195989b9-9d82-430b-b464-5e8928304bdb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/195989b9-9d82-430b-b464-5e8928304bdb\",\n      \"name\": \"195989b9-9d82-430b-b464-5e8928304bdb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-04-24T05:01:33.488341598Z\",\n      \"endTime\": \"2026-04-24T05:02:10.881244554Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0001bfaa0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-xv5wb2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/35394bfc-4a06-43b3-aa53-a1d60828c361\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/35394bfc-4a06-43b3-aa53-a1d60828c361\",\n      \"name\": \"35394bfc-4a06-43b3-aa53-a1d60828c361\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-04-22T09:47:15.284582065Z\",\n      \"endTime\": \"2026-04-22T09:48:23.845505796Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0001bfaa0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-xv5wb2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/35394bfc-4a06-43b3-aa53-a1d60828c361\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/35394bfc-4a06-43b3-aa53-a1d60828c361\",\n      \"name\": \"35394bfc-4a06-43b3-aa53-a1d60828c361\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-04-22T09:47:15.284582065Z\",\n      \"endTime\": \"2026-04-22T09:48:23.845505796Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047534526858530816","occurred_at":"2026-04-24T04:34:07Z","signature_id":"6c0108a49b790e72d13a010db752444cef14c3dc181ef82e5a4f8965cf12b337","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046879318772027392","occurred_at":"2026-04-22T09:10:33Z","signature_id":"200e82b61130b0e496380db62fba3d48341980e2169169844b87db2b8a56e38f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":11.11111111111111,"anchor_occurrences":2,"after_last_push_count":62,"trend_counts":[0,1,0,0,0,0,0,1],"trend_range":"2026-04-19..2026-04-26 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-30","2026-04-06"],"prior_runs_affected":35,"prior_last_seen_at":"2026-04-12T07:50:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: Interrupted by User","fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:35]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:35]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046046207125491712","occurred_at":"2026-04-20T02:00:04Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"443d5bf877a5a8072106dcd7ee154f7ec5cf70e7883b77cc29306405e54f0131","failure_pattern":"Process did not finish before 4h0m0s timeout","search_query":"Process did not finish before 4h0m0s timeout","failed_at":"unknown","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":11.11111111111111,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-04-19..2026-04-26 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"unknown","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":2}],"full_error_samples":["{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 4h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-27T02:36:51Z\"}","{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 4h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-26T06:00:09Z\"}"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048531755698229248","occurred_at":"2026-04-26T22:36:45Z","signature_id":"e866a7b39ddf32d1bf58138de6de78cb4986cc8b8b5bb67431025636d69fa4bb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048220527406878720","occurred_at":"2026-04-26T02:00:02Z","signature_id":"e866a7b39ddf32d1bf58138de6de78cb4986cc8b8b5bb67431025636d69fa4bb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"cluster-zstream-4-20-lmh9qb\" in resourcegroup=\"rg-zstream-upgrade-4-20-pjcrzv\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-lmh9qb in resource group rg-zstream-upgrade-4-20-pjcrzv, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.555555555555555,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,2,0,0],"trend_range":"2026-04-19..2026-04-26 UTC","prior_windows_present":4,"prior_window_starts":["2026-03-23","2026-03-30","2026-04-06","2026-04-13"],"prior_runs_affected":21,"prior_last_seen_at":"2026-04-16T02:00:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000cfd4d0\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-20-lmh9qb\" in resourcegroup=\"rg-zstream-upgrade-4-20-pjcrzv\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-lmh9qb in resource group rg-zstream-upgrade-4-20-pjcrzv, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000cfd4d0\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-20-lmh9qb\" in resourcegroup=\"rg-zstream-upgrade-4-20-pjcrzv\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-lmh9qb in resource group rg-zstream-upgrade-4-20-pjcrzv, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:94]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000907200\u003e: \n    failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-9b6btl\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-9b6btl, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:94]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000907200\u003e: \n    failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-9b6btl\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-9b6btl, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"7ea808d7a2819dfbbb0955a17c09815dfdeabfda6098193eb2435310de4389d7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"9bf0b88b14f49ca8939e47c5f49f7451c2fe89b52d6654d0baaf34e317690c40","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"ae6a7f7d2e3c3444ed20150a67378f751586e201de3a927f677f72a871b2bf44","failure_pattern":"DeleteResourceGroup timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-xs4bxm, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.555555555555555,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-04-19..2026-04-26 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:143]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000d74480\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-xs4bxm, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:143]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000d74480\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-xs4bxm, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"d5331347e2faf79329c3cdafc6a44e8ec7df1e1af045fc0133e9a7f3146e64a0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"66bc8d9427d72c6b659d24636018e42338a779f44749a6537f189769d48e7cd4","failure_pattern":"ERROR CODE: ConflictingConcurrentWriteNotAllowed; detail message The operation was interrupted by a conflicting concurrent write on the same entity. Please retry later.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ConflictingConcurrentWriteNotAllowed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.555555555555555,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-04-19..2026-04-26 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:249]: Unexpected error:\n    \u003c*errors.joinError | 0xc00171e978\u003e: \n    failed to cleanup resource group: ordered cleanup workflow failed for disabled-image-registry-l5b2fs: Delete non-networking resources: failed deleting disabled-image-registry-hcp-cluster (Microsoft.RedHatOpenShift/HCPOpenShiftClusters): DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-l5b2fs/providers/Microsoft.RedHatOpenShift/HCPOpenShiftClusters/disabled-image-registry-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: ConflictingConcurrentWriteNotAllowed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ConflictingConcurrentWriteNotAllowed\",\n        \"message\": \"The operation was interrupted by a conflicting concurrent write on the same entity. Please retry later.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:249]: Unexpected error:\n    \u003c*errors.joinError | 0xc00171e978\u003e: \n    failed to cleanup resource group: ordered cleanup workflow failed for disabled-image-registry-l5b2fs: Delete non-networking resources: failed deleting disabled-image-registry-hcp-cluster (Microsoft.RedHatOpenShift/HCPOpenShiftClusters): DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-l5b2fs/providers/Microsoft.RedHatOpenShift/HCPOpenShiftClusters/disabled-image-registry-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: ConflictingConcurrentWriteNotAllowed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ConflictingConcurrentWriteNotAllowed\",\n        \"message\": \"The operation was interrupted by a conflicting concurrent write on the same entity. Please retry later.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046956383668015104","occurred_at":"2026-04-22T14:16:47Z","signature_id":"04e67810388081d177ee7c2235f817b3562a0317dade3675b8b04ad92e0e8fa0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f27b30bef0ebb941cd79b083d3eb5ff3005b4b85f92582e4349d8748b1eaeb94","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9fbgg8\" to finish deleting: context canceled","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.555555555555555,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-04-19..2026-04-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-23"],"prior_runs_affected":2,"prior_last_seen_at":"2026-03-26T14:45:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:250] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000013098\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9fbgg8\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:250] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000013098\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9fbgg8\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"d9dc6aa889ea69ad197f41ed2afe30f8cfc2a7d6da44938a3e4e287480e4c19e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.555555555555555,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-04-19..2026-04-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-23"],"prior_runs_affected":2,"prior_last_seen_at":"2026-03-26T06:35:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000a5a780\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-xsgg2.apps.aro.b7c9d0c5o8o2d7a.ldkt.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000a5a780\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-xsgg2.apps.aro.b7c9d0c5o8o2d7a.ldkt.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","signature_id":"c1a77945ba81ac7509d545df31cfac708dc285cf96391246d2a02928ddb6d90d","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}