{"meta":{"start_date":"2026-04-26","end_date":"2026-04-26","start_at":"2026-04-26T00:00:00Z","end_at":"2026-04-27T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T14:04:59Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":3,"failed_runs":3,"raw_occurrences":39,"matched_occurrences":39,"runs_affected":3},"rows":[{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-vg9wz2, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-vg9wz2\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-vg9wz2, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":32,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":32,"after_last_push_count":32,"trend_counts":[0,0,0,0,0,0,32],"trend_range":"2026-04-20..2026-04-26 UTC","prior_windows_present":4,"prior_window_starts":["2026-03-23","2026-03-30","2026-04-06","2026-04-13"],"prior_runs_affected":34,"prior_last_seen_at":"2026-04-18T02:00:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00160e780\u003e: \n    failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-lv6fnh, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-lv6fnh\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-lv6fnh, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00160e780\u003e: \n    failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-lv6fnh, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-lv6fnh\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-lv6fnh, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009345d0\u003e: \n    failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-whhpxz, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-whhpxz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-whhpxz, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009345d0\u003e: \n    failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-whhpxz, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-whhpxz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-whhpxz, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000cbe3f0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-9rmffv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-9rmffv, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000cbe3f0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-9rmffv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-9rmffv, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"091379906f72b72c8716e255ca6a7ab5abeb5afe0a5154711f537ee6cccc7078","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"0abd5b447ee698b81f1707bd60ea9990ea759efcdcc972cf3200911f32951554","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"0c8eec4b448c989a4426a74b72aec2657e4d062e64c6bcd96691dd7a2d931c53","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"0ef18c5a73ad99cb3a2a7b35af140ba5b406753a710159114955666b9eca00ba","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"19376306f2cb2f8d3a34c840f2615729877d1101b37d66d169dab449359e4083","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"25641567eea06172ba6cb92bc7b2653b7a13b7ed8ec30d6bcb8671bfe48ab0eb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"337b3bcdf6bff71a570478cc431d07e080c480b6619d28bca13262e47613ca09","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"389f53eb08aaa262a4746d3d9df1c71749c4472ab81180e47886dd1718821289","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"399a40257d75cb5ffddcf870d190df6c60fe7d174ebdafc2970ef1225057d307","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"41b08b36509628977e0c4a29fd18cd907ffd9d873a916d8fc61ba0adc7ec6791","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"490e707732656bb205f0c2b7842ed396d133b31b4a489da5bef99790832fc47c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"4fce41b3b93955932fa55116f097b4af201d8d1bb7be911e1457047c023e9dd2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"6b842c3a6c5bf58fbc97686095bcbf0409ea27c7e6451d4f902c8f1e5041f639","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"7e150d909f159be9b5b28c44d39dacb5c4ae256b0edfd42c0e1b8f505520b27b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"7fabe0a0a77bc8b7d726cc32f55b76da2002a72646445f1a1380c793e15fa13f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"836f91f1cd8bfdcb0f96676459975cc2cb7c577ebb9e5319ec064f87ba28f637","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"878732d180af433b8995aec1e25accb54a0bd1f8571b38d54e9f2cab2e51f903","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"8b597f08f43ae682a1bdda5aad153f43e1c2aef983f36e3579154f9af1bb0201","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"93926f6c3672bf3976462b8daafe98f0cbc0830dd9baa34896cbedcf37cb7581","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"9bc6fd791bb6cbf6fab539a888a80559f0ce3d55ef74dc60926af2fe273e5834","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"b13db0749618eb55bb6b4ea4a674cd283003f154235f5b940a350eb0fbbaf2df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"ba662c0ce125a590e39c149172132dfe7ab95362572220bab74de87afa2b43ed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"bbf43f79c6a74e92b6455c17f090e462e7b43ce45f5c756585c890737ab9b260","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"c221e7013ea4fcac6bdf38d210d9546f99bedf0820ecea014803db00ce552805","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"d0f6120855a1b2832aceb656fedd514596f5d908f37c6256ec451b1b90c70e0d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"d5264602c365080c29189d40c4130bdc37966cee503cc99a25a11ce923a4428f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"e35e69fc71a500b0aea4b7c9fad71f2ca457959364405feab283d05411b76639","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"e451958423110a71d1ea4f2a18b5853a58b9708001e7b9aa92df7978027ca29a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"eb9783a4049e1fb83ac6d8bebfa5095eeeaebba84a9fa58a50e7ed1cbec762f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"ec50a6baede5053ee97065d9eeb3df2a36605205ba38b9ebfb42f3ac26c11e8c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"f84e9aa869e01832a7851d3aac2e566c7c9446050bb27e887945a04fad0709b2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"faa43504d0591365adc4ec1049ed25dc2a5ed93d482d2e94a7defc199479b1d4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"443d5bf877a5a8072106dcd7ee154f7ec5cf70e7883b77cc29306405e54f0131","failure_pattern":"Process did not finish before 4h0m0s timeout","search_query":"Process did not finish before 4h0m0s timeout","failed_at":"unknown","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":66.66666666666667,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-04-20..2026-04-26 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"unknown","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":2}],"full_error_samples":["{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 4h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-27T02:36:51Z\"}","{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 4h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-26T06:00:09Z\"}"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048531755698229248","occurred_at":"2026-04-26T22:36:45Z","signature_id":"e866a7b39ddf32d1bf58138de6de78cb4986cc8b8b5bb67431025636d69fa4bb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048220527406878720","occurred_at":"2026-04-26T02:00:02Z","signature_id":"e866a7b39ddf32d1bf58138de6de78cb4986cc8b8b5bb67431025636d69fa4bb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":2,"after_last_push_count":93,"trend_counts":[0,0,0,0,3,0,2],"trend_range":"2026-04-20..2026-04-26 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-23","2026-03-30","2026-04-06"],"prior_runs_affected":28,"prior_last_seen_at":"2026-04-10T10:59:58Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["onsi/ginkgo/v2.generateTable.func2.1({0xa0ccd80?, 0xc0009569c0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:293 +0x105 fp=0xc000b91f88 sp=0xc000b91f18 pc=0x3978245\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc000b91fe0 sp=0xc000b91f88 pc=0x3969bbb\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000b91fe8 sp=0xc000b91fe0 pc=0x34facc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 83\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\ngoroutine 179 gp=0xc000fa6fc0 m=nil [IO wait]:\nruntime.gopark(0xc000d914c0?, 0x44ab00e8?, 0x0?, 0x42?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000ed9720 sp=0xc000ed9700 pc=0x34f250e\nruntime.netpollblock(0x355cd18?, 0x347e066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc000ed9758 sp=0xc000ed9720 pc=0x34b1fb7\ninternal/poll.runtime_pollWait(0x7fd445b7d200, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc000ed9778 sp=0xc000ed9758 pc=0x34f0da5\ninternal/poll.(*pollDesc).wait(0xc00050fe00?, 0xc0012ac000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc000ed97a0 sp=0xc000ed9778 pc=0x357fa47\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc00050fe00, {0xc0012ac000, 0x3500, 0x3500})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc000ed9838 sp=0xc000ed97a0 pc=0x3580e79\nnet.(*netFD).Read(0xc00050fe00, {0xc0012ac000?, 0xc0012ac2cb?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc000ed9880 sp=0xc000ed9838 pc=0x37465e5\nnet.(*conn).Read(0xc0001a4a60, {0xc0012ac000?, 0x7fd43cfc2680?, 0x7fd48d7a55c0?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc000ed98c8 sp=0xc000ed9880 pc=0x3758685\ncrypto/tls.(*atLeastReader).Read(0xc00100a1e0, {0xc0012ac000?, 0x3230?, 0x34ef245?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc000ed9910 sp=0xc000ed98c8 pc=0x37c411b\nbytes.(*Buffer).ReadFrom(0xc000d1c628, {0x9feefe0, 0xc00100a1e0})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc000ed9968 sp=0xc000ed9910 pc=0x35bdc78\ncrypto/tls.(*Conn).readFromUntil(0xc000d1c388, {0x9fefb80, 0xc0001a4a60}, 0xc000ed99d0?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc000ed99a0 sp=0xc000ed9968 pc=0x37c42fe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc000d1c388, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc000ed9c18 sp=0xc000ed99a0 pc=0x37c137b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc000d1c388, {0xc001113000, 0x1000, 0x3bc0820?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc000ed9c80 sp=0xc000ed9c18 pc=0x37c7c65\nbufio.(*Reader).Read(0xc00061c9c0, {0xc000eea664, 0x9, 0x3bce8ae?})\n\t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0xc000ed9cb8 sp=0xc000ed9c80 pc=0x365a1d7\nio.ReadAtLeast({0x9fee5e0, 0xc00061c9c0}, {0xc000eea664, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0xc000ed9d00 sp=0xc000ed9cb8 pc=0x354ce8e\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc000eea664, 0x9, 0xc000000067?}, {0x9fee5e0?, 0xc00061c9c0?})\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:250 +0x65 fp=0xc000ed9d50 sp=0xc000ed9d00 pc=0x3bb75e5\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc000eea620)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:513 +0x6b fp=0xc000ed9dd8 sp=0xc000ed9d50 pc=0x3bb81ab\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc000eea620)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:572 +0x18 fp=0xc000ed9e18 sp=0xc000ed9dd8 pc=0x3bb8918\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000ed9fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2251 +0xca fp=0xc000ed9f60 sp=0xc000ed9e18 pc=0x3bcb7ea\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc000fa7500)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2120 +0x52 fp=0xc000ed9fc8 sp=0xc000ed9f60 pc=0x3bcab72\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0x25 fp=0xc000ed9fe0 sp=0xc000ed9fc8 pc=0x3bc4305\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000ed9fe8 sp=0xc000ed9fe0 pc=0x34facc1\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 178\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0xda5\n\nrax    0xca\nrbx    0x0\nrcx    0x34fcae3\nrdx    0x0\nrdi    0xdb9fcb0\nrsi    0x80\nrbp    0x7fff160bed10\nrsp    0x7fff160becc8\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x12\nr13    0x1\nr14    0xdb9dbc0\nr15    0x5\nrip    0x34fcae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command\nonsi/ginkgo/v2.generateTable.func2.1({0xa0ccd80?, 0xc0009569c0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:293 +0x105 fp=0xc000b91f88 sp=0xc000b91f18 pc=0x3978245\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc000b91fe0 sp=0xc000b91f88 pc=0x3969bbb\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000b91fe8 sp=0xc000b91fe0 pc=0x34facc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 83\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\ngoroutine 179 gp=0xc000fa6fc0 m=nil [IO wait]:\nruntime.gopark(0xc000d914c0?, 0x44ab00e8?, 0x0?, 0x42?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000ed9720 sp=0xc000ed9700 pc=0x34f250e\nruntime.netpollblock(0x355cd18?, 0x347e066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc000ed9758 sp=0xc000ed9720 pc=0x34b1fb7\ninternal/poll.runtime_pollWait(0x7fd445b7d200, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc000ed9778 sp=0xc000ed9758 pc=0x34f0da5\ninternal/poll.(*pollDesc).wait(0xc00050fe00?, 0xc0012ac000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc000ed97a0 sp=0xc000ed9778 pc=0x357fa47\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc00050fe00, {0xc0012ac000, 0x3500, 0x3500})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc000ed9838 sp=0xc000ed97a0 pc=0x3580e79\nnet.(*netFD).Read(0xc00050fe00, {0xc0012ac000?, 0xc0012ac2cb?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc000ed9880 sp=0xc000ed9838 pc=0x37465e5\nnet.(*conn).Read(0xc0001a4a60, {0xc0012ac000?, 0x7fd43cfc2680?, 0x7fd48d7a55c0?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc000ed98c8 sp=0xc000ed9880 pc=0x3758685\ncrypto/tls.(*atLeastReader).Read(0xc00100a1e0, {0xc0012ac000?, 0x3230?, 0x34ef245?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc000ed9910 sp=0xc000ed98c8 pc=0x37c411b\nbytes.(*Buffer).ReadFrom(0xc000d1c628, {0x9feefe0, 0xc00100a1e0})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc000ed9968 sp=0xc000ed9910 pc=0x35bdc78\ncrypto/tls.(*Conn).readFromUntil(0xc000d1c388, {0x9fefb80, 0xc0001a4a60}, 0xc000ed99d0?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc000ed99a0 sp=0xc000ed9968 pc=0x37c42fe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc000d1c388, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc000ed9c18 sp=0xc000ed99a0 pc=0x37c137b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc000d1c388, {0xc001113000, 0x1000, 0x3bc0820?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc000ed9c80 sp=0xc000ed9c18 pc=0x37c7c65\nbufio.(*Reader).Read(0xc00061c9c0, {0xc000eea664, 0x9, 0x3bce8ae?})\n\t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0xc000ed9cb8 sp=0xc000ed9c80 pc=0x365a1d7\nio.ReadAtLeast({0x9fee5e0, 0xc00061c9c0}, {0xc000eea664, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0xc000ed9d00 sp=0xc000ed9cb8 pc=0x354ce8e\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc000eea664, 0x9, 0xc000000067?}, {0x9fee5e0?, 0xc00061c9c0?})\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:250 +0x65 fp=0xc000ed9d50 sp=0xc000ed9d00 pc=0x3bb75e5\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc000eea620)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:513 +0x6b fp=0xc000ed9dd8 sp=0xc000ed9d50 pc=0x3bb81ab\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc000eea620)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:572 +0x18 fp=0xc000ed9e18 sp=0xc000ed9dd8 pc=0x3bb8918\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000ed9fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2251 +0xca fp=0xc000ed9f60 sp=0xc000ed9e18 pc=0x3bcb7ea\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc000fa7500)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2120 +0x52 fp=0xc000ed9fc8 sp=0xc000ed9f60 pc=0x3bcab72\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0x25 fp=0xc000ed9fe0 sp=0xc000ed9fc8 pc=0x3bc4305\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000ed9fe8 sp=0xc000ed9fe0 pc=0x34facc1\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 178\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0xda5\n\nrax    0xca\nrbx    0x0\nrcx    0x34fcae3\nrdx    0x0\nrdi    0xdb9fcb0\nrsi    0x80\nrbp    0x7fff160bed10\nrsp    0x7fff160becc8\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x12\nr13    0x1\nr14    0xdb9dbc0\nr15    0x5\nrip    0x34fcae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command","/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:199\n          \u003e github.com/onsi/ginkgo/v2/internal.(*group).run(0xc000e24b88, {0xc000b26620?, 0x0?, 0x0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:349\n          \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).runSpecs(0xc000a4aa88, {0x0, 0x0}, {0xdbcaa00, 0x0, 0x0}, {0xc0000be094, 0x2a}, 0x0, {0xc000b26600, ...})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:496\n          \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).RunSpec(_, {_, _}, {_, _, _}, {_, _}, {0xc0000be094, 0x2a}, ...)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite_patch.go:68\n            github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite.func1.1({0x3481a85?, 0xc000b25980?})\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/ginkgo/util.go:85\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.runSpec({0xa0b0da8, 0xc00060dcc0}, 0xc000d7e000, 0x1)\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:596\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:237\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:230\n\n          goroutine 53 [select]\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2(0xc000d7e000?)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:131\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts in goroutine 94\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:128\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:115 @ 04/26/26 14:20:43.171\n  \"ts\"=\"2026-04-26 14:20:43.172175\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-26 14:20:43.172316\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-candidate-4-21-w8wp9g-l44gtn\" \"resourceGroup\"=\"rg-candidate-4-21-nl8wck\"\n  \"ts\"=\"2026-04-26 14:20:43.370310\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-candidate-4-21-w8wp9g-l44gtn\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-26 14:20:44.473602\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-6427b6424c4038306dc527fd5f7fc0b377fe9c26e56432adfe480dff6189e0c7.yaml.gz\"\n  \"ts\"=\"2026-04-26 14:20:44.473758\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-6427b6424c4038306dc527fd5f7fc0b377fe9c26e56432adfe480dff6189e0c7.yaml.gz\"\n  \"ts\"=\"2026-04-26 14:20:44.473903\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-26 14:20:44.476357\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-117\"\n  \"ts\"=\"2026-04-26 14:20:44.476386\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-candidate-4-21-nl8wck\"\n  \"ts\"=\"2026-04-26 14:20:44.634596\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-117\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-26 14:20:44.648345\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-candidate-4-21-nl8wck\" \"deployment\"=\"pooledNonMsiScopedAssignments-hdyao36g42hfg\"\n  \"ts\"=\"2026-04-26 14:20:44.765715\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-117\" \"deployment\"=\"msi-deployment-117\"\n  \"ts\"=\"2026-04-26 14:20:44.786703\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-candidate-4-21-nl8wck\" \"deployment\"=\"customer-infra-cluster-candidate-4-21-w8wp9g-l44gtn\"\n  \"ts\"=\"2026-04-26 14:20:45.009501\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-26 14:20:45.327005\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-26 14:20:45.327061\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-candidate-4-21-nl8wck\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:199\n          \u003e github.com/onsi/ginkgo/v2/internal.(*group).run(0xc000e24b88, {0xc000b26620?, 0x0?, 0x0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:349\n          \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).runSpecs(0xc000a4aa88, {0x0, 0x0}, {0xdbcaa00, 0x0, 0x0}, {0xc0000be094, 0x2a}, 0x0, {0xc000b26600, ...})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:496\n          \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).RunSpec(_, {_, _}, {_, _, _}, {_, _}, {0xc0000be094, 0x2a}, ...)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite_patch.go:68\n            github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite.func1.1({0x3481a85?, 0xc000b25980?})\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/ginkgo/util.go:85\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.runSpec({0xa0b0da8, 0xc00060dcc0}, 0xc000d7e000, 0x1)\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:596\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:237\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:230\n\n          goroutine 53 [select]\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2(0xc000d7e000?)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:131\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts in goroutine 94\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:128\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:115 @ 04/26/26 14:20:43.171\n  \"ts\"=\"2026-04-26 14:20:43.172175\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-26 14:20:43.172316\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-candidate-4-21-w8wp9g-l44gtn\" \"resourceGroup\"=\"rg-candidate-4-21-nl8wck\"\n  \"ts\"=\"2026-04-26 14:20:43.370310\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-candidate-4-21-w8wp9g-l44gtn\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-26 14:20:44.473602\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-6427b6424c4038306dc527fd5f7fc0b377fe9c26e56432adfe480dff6189e0c7.yaml.gz\"\n  \"ts\"=\"2026-04-26 14:20:44.473758\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-6427b6424c4038306dc527fd5f7fc0b377fe9c26e56432adfe480dff6189e0c7.yaml.gz\"\n  \"ts\"=\"2026-04-26 14:20:44.473903\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-26 14:20:44.476357\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-117\"\n  \"ts\"=\"2026-04-26 14:20:44.476386\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-candidate-4-21-nl8wck\"\n  \"ts\"=\"2026-04-26 14:20:44.634596\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-117\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-26 14:20:44.648345\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-candidate-4-21-nl8wck\" \"deployment\"=\"pooledNonMsiScopedAssignments-hdyao36g42hfg\"\n  \"ts\"=\"2026-04-26 14:20:44.765715\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-117\" \"deployment\"=\"msi-deployment-117\"\n  \"ts\"=\"2026-04-26 14:20:44.786703\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-candidate-4-21-nl8wck\" \"deployment\"=\"customer-infra-cluster-candidate-4-21-w8wp9g-l44gtn\"\n  \"ts\"=\"2026-04-26 14:20:45.009501\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-26 14:20:45.327005\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-26 14:20:45.327061\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-candidate-4-21-nl8wck\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"135a8d090682264ff405d4580f350f5b62f5f9deea9dbb6faf885ee973d302b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"e2b4b5977905781ce553eb3fff22f33d157c0295fcd95cc56c9f33a8695c46dd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e9c6c185c09aa6c23477c53c1d3d7fb1bb60e041c3cb718558ce145b6718d4d","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":6,"trend_counts":[0,0,0,1,1,1,1],"trend_range":"2026-04-20..2026-04-26 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-30","2026-04-06"],"prior_runs_affected":6,"prior_last_seen_at":"2026-04-10T02:06:29Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.000s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.000s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"8a441c3cc9a1724fd565ecc68f397cdfdb078c056f41b0ab5661bd4e719eb328","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":62,"trend_counts":[1,0,0,0,0,0,1],"trend_range":"2026-04-20..2026-04-26 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-30","2026-04-06"],"prior_runs_affected":35,"prior_last_seen_at":"2026-04-12T07:50:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"9626adf85231cae7571909c00084bdfb7c9de3df3208a3d5200ee95baf8fc7e8","failure_pattern":"context deadline exceeded","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,1,0,1,1],"trend_range":"2026-04-20..2026-04-26 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: Unexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: Unexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2048369023409721344","occurred_at":"2026-04-26T11:50:06Z","signature_id":"b15c55d32190fb3d32f16f58a52f080055a52cddf394a6cd7e062c8467f788e4","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}