{"meta":{"start_date":"2026-04-30","end_date":"2026-04-30","start_at":"2026-04-30T00:00:00Z","end_at":"2026-05-01T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T15:28:19Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":8,"failed_runs":7,"raw_occurrences":46,"matched_occurrences":46,"runs_affected":7},"rows":[{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-ngvljq, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-ngvljq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-ngvljq, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":33,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":33,"after_last_push_count":33,"trend_counts":[0,0,0,0,0,0,33],"trend_range":"2026-04-24..2026-04-30 UTC","prior_windows_present":4,"prior_window_starts":["2026-03-30","2026-04-06","2026-04-13","2026-04-20"],"prior_runs_affected":32,"prior_last_seen_at":"2026-04-26T11:50:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000d3e6c0\u003e: \n    failed to create HCP cluster negative-tests-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-d8cxjg, error: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-d8cxjg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-d8cxjg, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000d3e6c0\u003e: \n    failed to create HCP cluster negative-tests-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-d8cxjg, error: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-d8cxjg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-d8cxjg, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001180de0\u003e: \n    failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-5z782n, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-5z782n\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-5z782n, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001180de0\u003e: \n    failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-5z782n, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-5z782n\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-5z782n, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:126]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e0cab0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-xvrhv8, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-xvrhv8 in resource group rg-cp-ystream-upgrade-4-21-znfmjs, error: failed waiting for cluster=\"cp-ystream-upgrade-4-21-xvrhv8\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-znfmjs\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-xvrhv8 in resource group rg-cp-ystream-upgrade-4-21-znfmjs, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:126]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e0cab0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-xvrhv8, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-xvrhv8 in resource group rg-cp-ystream-upgrade-4-21-znfmjs, error: failed waiting for cluster=\"cp-ystream-upgrade-4-21-xvrhv8\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-znfmjs\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-xvrhv8 in resource group rg-cp-ystream-upgrade-4-21-znfmjs, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"05468eab2a133563f11fe8ddf8ef0da8f39cf4d82f7dd3e167feb372b0b7b043","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"09c8898a516665064cab3818eb1a3c57d6f05835c48f747a805b5a095c376981","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"1574ecbd4c56c11b45a30ed362353c1e98cc5f5e1a3b1a779ce82a83841d4a0b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"193a7929dee01036fe369834497cdcc5c6f4a7644dcd76449f7199b046afa8fd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"1945093724d17deff67b196ee9a2e00cd7d1f581ee2867caf113115d03a7d8ed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"280a0af286920eaa2b5e78c363498f3ff14c333deb4158a213c74eb84871af96","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"2a22d3180c7b1f48df438866c9d6920322532c9e8f7d70c649c5110793442d38","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"46f9b2cb9bbc216ac4a7ebbcb1944b9489948a5ce5b5b49f0019cce2136b6cb0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"4b89f46da008622bde1a3148ce39c2125654581b34ce80b0889bba2edf6077d9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"4d8f2ea99b53f14ac593e17954338a9b482b690c9fe115e2843c15c71e1a9a96","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"50bc0cdbd532066c59fff87615b5ba41f0dc362aeeb8fbeea2e93a1d9c0aab69","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"50f4bdb500d654d5d442355f0221f497291ce0994d5f6d7166b3ae2065857bcf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"5bf3cfc15497c0352a1280c377c10364b5b490e5f11f58d7bdcb3d909937b649","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"60cb6882a4e55a64e88c8c6b603dfea8a0dded3f09bd002a362d114c43b063a6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"787a2852bb7f0884270266de3fac1edc05085510941a93fa29dde1a7a78e6b5e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"7968ff9c3a313c7e352e04383b761dc38b6273e812b58cf39814413fed41fbfe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"7a649cfbe4bd20e16fef05dac58f75e138dc8bc338b45bca40dfbfd188390e02","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"7d5f6556261668d74558aa2b7f9ac9c21a4cf0092259932c278b2e43969136a0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"819aafbed488f1f499d88b6c7ebbf81c523831453a8963055d1e0801eb767272","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"97ac1f64ac6c72d28f43dcd1df785c295489b69f91d5be460454efa52d4368d2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"97f904595b7c396f290ba524eb49e9d09e1f6df4c2ee33a702db1d1c2c7d1930","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"9cfc0e78f4d913dfbd43b2ac530d75e00939ec0ff1610c8cc8fea4d77b9c9da6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"9d03b9805042d6f01a2997204cf84abebdd2c68f253f1ef5fca5e09217e3fc55","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"a453fac907798a5a292190a4d5f1d54ce29da720497b222b11b99a1f68ddc268","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"b809190d07a3ea8893c768cf92228fbef3126678fce1037116879d7221a13d6a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"c753a1085cdceb8bff7eba07576a0c3fca121bbb8022d803d5bbc8377c06fbde","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"dcc912620021d70fed1899cbcc7ea033a1c001b21d2d2c5d1e45f097e39962e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"e0a0a9289c44038052c3f89a1b5f05aca095dfee0db8cdbff00a29c3a74c4e69","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"ee7f4f40f887107747cd0ff6cb2940a0c7401af6a94423558e6d8f1026c002ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"fadb259f7262ee1699b11a4a9028fb5809ab3f009b6fde80e195569e7026d605","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"fb2f84e193da6e827a91b7a8ac6badc6960739c09bac82279a22f11b3f9896e7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"fc963b82122d66cc957c47f6701e0a351f57ea40ad6f81a7a370b7de73a5ff5c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"fdeb0461a5362ca1d4de5f02f7922f743bf9d2f4f8e446959be529af30dc667f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f7692883b7d8edf2d0661368bcbf0596b25bb01a118dcf71cc62820145611635","failure_pattern":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set: assertion failed: expected values to equal","search_query":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set: assertion failed: expected values to equal","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":87.5,"anchor_occurrences":7,"after_last_push_count":22,"trend_counts":[3,1,0,0,1,7,7],"trend_range":"2026-04-24..2026-04-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-04-20"],"prior_runs_affected":22,"prior_last_seen_at":"2026-04-25T01:00:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set","occurrences":6},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/mise_routing.go:89]: Expected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/mise_routing.go:89]: Expected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049861090002604032","occurred_at":"2026-04-30T14:39:02Z","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049736957507932160","occurred_at":"2026-04-30T06:25:47Z","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049726395587760128","occurred_at":"2026-04-30T05:43:49Z","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049689436916027392","occurred_at":"2026-04-30T03:16:57Z","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049681170710401024","occurred_at":"2026-04-30T02:44:06Z","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049657136199241728","occurred_at":"2026-04-30T01:08:21Z","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":2,"after_last_push_count":76,"trend_counts":[3,0,2,0,1,7,2],"trend_range":"2026-04-24..2026-04-30 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-30","2026-04-06","2026-04-20"],"prior_runs_affected":30,"prior_last_seen_at":"2026-04-26T11:50:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["}, 0xc000091c20, {0xc000da6270, 0x22}, {0x82e3522, 0xd}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:852\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0xc0015ce190, 0x7}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:346\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func12.1({0x7f76acf97ee0, 0xc000fdca80})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:99\n                | \n                | By(\"seeding HCP cluster\")\n                \u003e err = tc.CreateHCPClusterFromParam(\n                | \tctx,\n                | \tGinkgoLogr,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8e229a0?, 0xc000fdca80?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 88\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106 @ 04/30/26 04:31:10.931\n  \"ts\"=\"2026-04-30 04:31:10.932793\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-30 04:31:10.932937\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-subnet-reuse-h8ngz8\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-8nwq6c\"\n  \"ts\"=\"2026-04-30 04:31:11.270334\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-cluster-hcv4ln\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-8nwq6c\"\n  \"ts\"=\"2026-04-30 04:31:11.634642\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-subnet-reuse-h8ngz8\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-30 04:31:12.993616\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-cluster-hcv4ln\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-30 04:31:14.099138\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-6950b6d79a3c274e15675866beef4b6fc9a309f9234c1ea3a6f0c13bb6db0b9f.yaml.gz\"\n  \"ts\"=\"2026-04-30 04:31:14.099274\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-6950b6d79a3c274e15675866beef4b6fc9a309f9234c1ea3a6f0c13bb6db0b9f.yaml.gz\"\n  \"ts\"=\"2026-04-30 04:31:14.099443\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-30 04:31:14.101343\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-11\"\n  \"ts\"=\"2026-04-30 04:31:14.101495\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-8nwq6c\"\n  \"ts\"=\"2026-04-30 04:31:14.101531\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-83\"\n  \"ts\"=\"2026-04-30 04:31:14.257439\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-11\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-30 04:31:14.323011\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-8nwq6c\" \"deployment\"=\"pooledNonMsiScopedAssignments-j3gkygvtvkvjs\"\n  \"ts\"=\"2026-04-30 04:31:14.385540\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-11\" \"deployment\"=\"msi-deployment-11\"\n  \"ts\"=\"2026-04-30 04:31:14.421103\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-83\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-30 04:31:14.462928\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-8nwq6c\" \"deployment\"=\"pooledNonMsiScopedAssignments-67tmrjj4do2cc\"\n  \"ts\"=\"2026-04-30 04:31:14.559183\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-83\" \"deployment\"=\"msi-deployment-83\"\n  \"ts\"=\"2026-04-30 04:31:14.617918\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-8nwq6c\" \"deployment\"=\"customer-infra-basic-cluster-hcv4ln\"\n  \"ts\"=\"2026-04-30 04:31:14.790654\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-8nwq6c\" \"deployment\"=\"customer-infra-cluster-subnet-reuse-h8ngz8\"\n  \"ts\"=\"2026-04-30 04:31:14.933407\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-30 04:31:15.397732\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-30 04:31:15.397999\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-8nwq6c\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n}, 0xc000091c20, {0xc000da6270, 0x22}, {0x82e3522, 0xd}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:852\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0xc0015ce190, 0x7}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:346\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func12.1({0x7f76acf97ee0, 0xc000fdca80})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:99\n                | \n                | By(\"seeding HCP cluster\")\n                \u003e err = tc.CreateHCPClusterFromParam(\n                | \tctx,\n                | \tGinkgoLogr,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8e229a0?, 0xc000fdca80?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 88\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106 @ 04/30/26 04:31:10.931\n  \"ts\"=\"2026-04-30 04:31:10.932793\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-30 04:31:10.932937\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-subnet-reuse-h8ngz8\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-8nwq6c\"\n  \"ts\"=\"2026-04-30 04:31:11.270334\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-cluster-hcv4ln\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-8nwq6c\"\n  \"ts\"=\"2026-04-30 04:31:11.634642\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-subnet-reuse-h8ngz8\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-30 04:31:12.993616\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-cluster-hcv4ln\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-30 04:31:14.099138\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-6950b6d79a3c274e15675866beef4b6fc9a309f9234c1ea3a6f0c13bb6db0b9f.yaml.gz\"\n  \"ts\"=\"2026-04-30 04:31:14.099274\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-6950b6d79a3c274e15675866beef4b6fc9a309f9234c1ea3a6f0c13bb6db0b9f.yaml.gz\"\n  \"ts\"=\"2026-04-30 04:31:14.099443\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-30 04:31:14.101343\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-11\"\n  \"ts\"=\"2026-04-30 04:31:14.101495\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-8nwq6c\"\n  \"ts\"=\"2026-04-30 04:31:14.101531\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-83\"\n  \"ts\"=\"2026-04-30 04:31:14.257439\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-11\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-30 04:31:14.323011\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-8nwq6c\" \"deployment\"=\"pooledNonMsiScopedAssignments-j3gkygvtvkvjs\"\n  \"ts\"=\"2026-04-30 04:31:14.385540\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-11\" \"deployment\"=\"msi-deployment-11\"\n  \"ts\"=\"2026-04-30 04:31:14.421103\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-83\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-30 04:31:14.462928\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-8nwq6c\" \"deployment\"=\"pooledNonMsiScopedAssignments-67tmrjj4do2cc\"\n  \"ts\"=\"2026-04-30 04:31:14.559183\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-83\" \"deployment\"=\"msi-deployment-83\"\n  \"ts\"=\"2026-04-30 04:31:14.617918\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-8nwq6c\" \"deployment\"=\"customer-infra-basic-cluster-hcv4ln\"\n  \"ts\"=\"2026-04-30 04:31:14.790654\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-8nwq6c\" \"deployment\"=\"customer-infra-cluster-subnet-reuse-h8ngz8\"\n  \"ts\"=\"2026-04-30 04:31:14.933407\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-30 04:31:15.397732\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-30 04:31:15.397999\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-8nwq6c\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","fb78 pc=0x6c97dce\ngithub.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:237 +0x196 fp=0xc000deffe0 sp=0xc000defc20 pc=0x6c93b36\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000deffe8 sp=0xc000deffe0 pc=0x2dc5cc1\ncreated by github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:230 +0x1e5\n\ngoroutine 156 gp=0xc000df9a40 m=nil [IO wait]:\nruntime.gopark(0xc0001487e0?, 0x2d7ae8b?, 0x0?, 0x98?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000e92720 sp=0xc000e92700 pc=0x2dbd50e\nruntime.netpollblock(0x2e27d18?, 0x2d49066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc000e92758 sp=0xc000e92720 pc=0x2d7cfb7\ninternal/poll.runtime_pollWait(0x7f49cf58b000, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc000e92778 sp=0xc000e92758 pc=0x2dbbda5\ninternal/poll.(*pollDesc).wait(0xc000626c00?, 0xc001438000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc000e927a0 sp=0xc000e92778 pc=0x2e4aa47\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc000626c00, {0xc001438000, 0x3500, 0x3500})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc000e92838 sp=0xc000e927a0 pc=0x2e4be79\nnet.(*netFD).Read(0xc000626c00, {0xc001438000?, 0xc0014382c9?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc000e92880 sp=0xc000e92838 pc=0x300f705\nnet.(*conn).Read(0xc0000b6bc8, {0xc001438000?, 0x7f49cc15bf28?, 0x7f4a161705c0?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc000e928c8 sp=0xc000e92880 pc=0x30217a5\ncrypto/tls.(*atLeastReader).Read(0xc000f32018, {0xc001438000?, 0x3232?, 0x2dba245?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc000e92910 sp=0xc000e928c8 pc=0x308d23b\nbytes.(*Buffer).ReadFrom(0xc000d64d28, {0x8d773c0, 0xc000f32018})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc000e92968 sp=0xc000e92910 pc=0x2e88c38\ncrypto/tls.(*Conn).readFromUntil(0xc000d64a88, {0x8d77e60, 0xc0000b6bc8}, 0xc000e929d0?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc000e929a0 sp=0xc000e92968 pc=0x308d41e\ncrypto/tls.(*Conn).readRecordOrCCS(0xc000d64a88, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc000e92c18 sp=0xc000e929a0 pc=0x308a49b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc000d64a88, {0xc001019000, 0x1000, 0x348a760?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc000e92c80 sp=0xc000e92c18 pc=0x3090d85\nbufio.(*Reader).Read(0xc000dbf6e0, {0xc000cfc204, 0x9, 0x34987ee?})\n\t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0xc000e92cb8 sp=0xc000e92c80 pc=0x2f232f7\nio.ReadAtLeast({0x8d76820, 0xc000dbf6e0}, {0xc000cfc204, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0xc000e92d00 sp=0xc000e92cb8 pc=0x2e17e8e\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc000cfc204, 0x9, 0xc0000000dd?}, {0x8d76820?, 0xc000dbf6e0?})\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:250 +0x65 fp=0xc000e92d50 sp=0xc000e92d00 pc=0x3481525\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc000cfc1c0)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:513 +0x6b fp=0xc000e92dd8 sp=0xc000e92d50 pc=0x34820eb\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc000cfc1c0)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:572 +0x18 fp=0xc000e92e18 sp=0xc000e92dd8 pc=0x3482858\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000e92fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2251 +0xca fp=0xc000e92f60 sp=0xc000e92e18 pc=0x349572a\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc0001ff6c0)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2120 +0x52 fp=0xc000e92fc8 sp=0xc000e92f60 pc=0x3494ab2\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0x25 fp=0xc000e92fe0 sp=0xc000e92fc8 pc=0x348e245\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000e92fe8 sp=0xc000e92fe0 pc=0x2dc5cc1\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 155\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0xda5\n\nrax    0xfffffffffffffffc\nrbx    0x0\nrcx    0x2dc7ae3\nrdx    0x0\nrdi    0xc2d83f0\nrsi    0x80\nrbp    0x7f49cf57ccb8\nrsp    0x7f49cf57cc70\nr8     0x0\nr9     0x0\nr10    0x7f49cf57cca8\nr11    0x202\nr12    0x7f49cf57cca8\nr13    0x1\nr14    0xc000002c40\nr15    0x0\nrip    0x2dc7ae3\nrflags 0x202\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command\nfb78 pc=0x6c97dce\ngithub.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:237 +0x196 fp=0xc000deffe0 sp=0xc000defc20 pc=0x6c93b36\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000deffe8 sp=0xc000deffe0 pc=0x2dc5cc1\ncreated by github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:230 +0x1e5\n\ngoroutine 156 gp=0xc000df9a40 m=nil [IO wait]:\nruntime.gopark(0xc0001487e0?, 0x2d7ae8b?, 0x0?, 0x98?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000e92720 sp=0xc000e92700 pc=0x2dbd50e\nruntime.netpollblock(0x2e27d18?, 0x2d49066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc000e92758 sp=0xc000e92720 pc=0x2d7cfb7\ninternal/poll.runtime_pollWait(0x7f49cf58b000, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc000e92778 sp=0xc000e92758 pc=0x2dbbda5\ninternal/poll.(*pollDesc).wait(0xc000626c00?, 0xc001438000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc000e927a0 sp=0xc000e92778 pc=0x2e4aa47\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc000626c00, {0xc001438000, 0x3500, 0x3500})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc000e92838 sp=0xc000e927a0 pc=0x2e4be79\nnet.(*netFD).Read(0xc000626c00, {0xc001438000?, 0xc0014382c9?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc000e92880 sp=0xc000e92838 pc=0x300f705\nnet.(*conn).Read(0xc0000b6bc8, {0xc001438000?, 0x7f49cc15bf28?, 0x7f4a161705c0?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc000e928c8 sp=0xc000e92880 pc=0x30217a5\ncrypto/tls.(*atLeastReader).Read(0xc000f32018, {0xc001438000?, 0x3232?, 0x2dba245?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc000e92910 sp=0xc000e928c8 pc=0x308d23b\nbytes.(*Buffer).ReadFrom(0xc000d64d28, {0x8d773c0, 0xc000f32018})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc000e92968 sp=0xc000e92910 pc=0x2e88c38\ncrypto/tls.(*Conn).readFromUntil(0xc000d64a88, {0x8d77e60, 0xc0000b6bc8}, 0xc000e929d0?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc000e929a0 sp=0xc000e92968 pc=0x308d41e\ncrypto/tls.(*Conn).readRecordOrCCS(0xc000d64a88, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc000e92c18 sp=0xc000e929a0 pc=0x308a49b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc000d64a88, {0xc001019000, 0x1000, 0x348a760?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc000e92c80 sp=0xc000e92c18 pc=0x3090d85\nbufio.(*Reader).Read(0xc000dbf6e0, {0xc000cfc204, 0x9, 0x34987ee?})\n\t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0xc000e92cb8 sp=0xc000e92c80 pc=0x2f232f7\nio.ReadAtLeast({0x8d76820, 0xc000dbf6e0}, {0xc000cfc204, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0xc000e92d00 sp=0xc000e92cb8 pc=0x2e17e8e\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc000cfc204, 0x9, 0xc0000000dd?}, {0x8d76820?, 0xc000dbf6e0?})\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:250 +0x65 fp=0xc000e92d50 sp=0xc000e92d00 pc=0x3481525\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc000cfc1c0)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:513 +0x6b fp=0xc000e92dd8 sp=0xc000e92d50 pc=0x34820eb\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc000cfc1c0)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:572 +0x18 fp=0xc000e92e18 sp=0xc000e92dd8 pc=0x3482858\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000e92fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2251 +0xca fp=0xc000e92f60 sp=0xc000e92e18 pc=0x349572a\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc0001ff6c0)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2120 +0x52 fp=0xc000e92fc8 sp=0xc000e92f60 pc=0x3494ab2\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0x25 fp=0xc000e92fe0 sp=0xc000e92fc8 pc=0x348e245\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000e92fe8 sp=0xc000e92fe0 pc=0x2dc5cc1\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 155\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0xda5\n\nrax    0xfffffffffffffffc\nrbx    0x0\nrcx    0x2dc7ae3\nrdx    0x0\nrdi    0xc2d83f0\nrsi    0x80\nrbp    0x7f49cf57ccb8\nrsp    0x7f49cf57cc70\nr8     0x0\nr9     0x0\nr10    0x7f49cf57cca8\nr11    0x202\nr12    0x7f49cf57cca8\nr13    0x1\nr14    0xc000002c40\nr15    0x0\nrip    0x2dc7ae3\nrflags 0x202\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"191f62852db4f448b87ba68c13a19e58c7092cd2cd1529136e47d7d4a7fb937c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"db9648e1714465c6b45e2c89a7e5c213f2fbafd18e2d92395fd33937fbb708ed","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e9c6c185c09aa6c23477c53c1d3d7fb1bb60e041c3cb718558ce145b6718d4d","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":1,"after_last_push_count":9,"trend_counts":[1,1,1,0,1,2,1],"trend_range":"2026-04-24..2026-04-30 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-30","2026-04-06","2026-04-20"],"prior_runs_affected":10,"prior_last_seen_at":"2026-04-26T11:50:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 2700.000s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 2700.000s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"8a441c3cc9a1724fd565ecc68f397cdfdb078c056f41b0ab5661bd4e719eb328","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-5xrsk8, error: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-5xrsk8\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-04-24..2026-04-30 UTC","prior_windows_present":3,"prior_window_starts":["2026-04-06","2026-04-13","2026-04-20"],"prior_runs_affected":9,"prior_last_seen_at":"2026-04-24T18:17:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:242]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001602a50\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-5xrsk8, error: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-5xrsk8\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:242]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001602a50\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-5xrsk8, error: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-5xrsk8\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049689436916027392","occurred_at":"2026-04-30T03:16:57Z","signature_id":"9783bb4694f754c68f5240df26e9864bf4c916a5b92dbc58a05a40888716ec0d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d78c0bca89c95459375355a5923414dd1603fcc7a1039f0264667a1b20c77dc3","failure_pattern":"ERROR CODE: DeploymentNotFound","search_query":"ERROR CODE: DeploymentNotFound","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-04-24..2026-04-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-30"],"prior_runs_affected":2,"prior_last_seen_at":"2026-04-02T18:07:57Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:81]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000e7aee0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cilium-cluster-8npjqv\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cilium-cluster-8npjqv/operationStatuses/08584240774378823233\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: DeploymentNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DeploymentNotFound\",\n        \"message\": \"Deployment 'mi-cilium-cluster-8npjqv' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:81]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000e7aee0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cilium-cluster-8npjqv\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cilium-cluster-8npjqv/operationStatuses/08584240774378823233\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: DeploymentNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DeploymentNotFound\",\n        \"message\": \"Deployment 'mi-cilium-cluster-8npjqv' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049726395587760128","occurred_at":"2026-04-30T05:43:49Z","signature_id":"d39a262db707d11054fd2d93431cb9020789ab0ccead0ea6cfb6d3cc5f56b9fa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"9626adf85231cae7571909c00084bdfb7c9de3df3208a3d5200ee95baf8fc7e8","failure_pattern":"context deadline exceeded","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":1,"after_last_push_count":7,"trend_counts":[0,1,1,0,1,2,1],"trend_range":"2026-04-24..2026-04-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-04-20"],"prior_runs_affected":7,"prior_last_seen_at":"2026-04-26T11:50:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: Unexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: Unexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049670252735762432","occurred_at":"2026-04-30T02:00:43Z","signature_id":"b15c55d32190fb3d32f16f58a52f080055a52cddf394a6cd7e062c8467f788e4","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}