{"meta":{"start_date":"2026-04-29","end_date":"2026-04-29","start_at":"2026-04-29T00:00:00Z","end_at":"2026-04-30T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T16:25:28Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":10,"failed_runs":10,"raw_occurrences":86,"matched_occurrences":83,"runs_affected":7},"rows":[{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-6ks5n8, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-6ks5n8\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-6ks5n8, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":61,"runs_affected":2,"failed_runs":2,"run_impact_percent":20,"anchor_occurrences":61,"after_last_push_count":266,"trend_counts":[6,23,35,32,0,34,61],"trend_range":"2026-04-23..2026-04-29 UTC","prior_windows_present":4,"prior_window_starts":["2026-03-30","2026-04-06","2026-04-13","2026-04-20"],"prior_runs_affected":31,"prior_last_seen_at":"2026-04-26T11:50:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009dbbc0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-ckbwst, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-ckbwst\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-ckbwst, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009dbbc0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-ckbwst, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-ckbwst\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-ckbwst, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014826f0\u003e: \n    failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-cwbjrq, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-cwbjrq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-cwbjrq, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014826f0\u003e: \n    failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-cwbjrq, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-cwbjrq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-cwbjrq, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00156e450\u003e: \n    failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ms8tn6, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-ms8tn6\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ms8tn6, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00156e450\u003e: \n    failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ms8tn6, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-ms8tn6\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ms8tn6, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"13903c76fb69b2344483336b8cb31eab8b38576ba98a0ec359e2f314a6232b68","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"22400cb6424b10bc08930c48e0d8c8f7b20e260b1b0983d3a3c79a3b36a2fb64","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"22de37dff543003872008c10bf0a71f9d3eb1f0fc88325d950d8baec35ea5287","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"2d8b65e3e93625382a8809e385e2e785e48a9f00eeba2088517ec9f8f450128e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"31b056d33fb3cfc6f7ec9799ad7468d87e5a9ff296cd3e285e45017bd3131fc7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"38ea6d6bb623fc9e1c03a4dc9a4703a17751ea0fc84b3161c2f8626df1261190","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"3e7ba2ac872311be803f46a230a3a729ad9934337ee2452608dbd1052a7fc61d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"3ed10e0a93958eff0f1083f09b28d87c89a00dfd06cf3fd7f91863d114efdb42","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"3ef92ee67a56828f77289d5cfa0dba16e4170a27a8df2f5439acf2dcde6bc305","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"4d0d11bdc293c0e58932314c542f0bec284656aaacb206d428bd006b6b69f2d1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"5949c13449ef29770ae9ca22de2760fd82a601d916e9b71f60c9fb5d28ffa636","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"5c7c1bdb299ed00f468d1693586d4b02a646ef01781b85942d7ad41019c9add5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"5ee0c1b9dede7001482a223923e5c6816c730c8a804135cad8a86f6175288f74","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"6deea125f9dbaafa2a071fa3dacc587fb9c0ebbd099115fb1395041e61fac803","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"6f55486917ff1587c8a353c487e9cb72802de4a63b413543ab17617ef418d5c2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"70792221d9f18c1a5ecfea4575243e59ffe8336e68af319786cc12c8dcdd55eb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"9b9aef1ffbe84836d553f174a21ad7a2c3e9e3deb2a96950c9ba868cca019073","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"a8ef4aaff1373745313df0a847e89f99d16fa9a805511c8c86f9ec75bc6e08dc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"a98f06ac0269dba04e334aa04102226d6ae00f94bec5f4e0a0e4898f8bb1ddf5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"aedd465c7fbda2a3f7384345f91617b0e7b61d5800ff2ca59311c7181b43b6ec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"b088c75d68851cadcf95dd0c6f45086504d3790f06cf6ac9c3b6667d6cffbde1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"b8375dc982d506dc907cc6309aa10e7c506225b9fa95cf1bc2669737b16cb0af","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"b8416b5a38752f0be34f1e410acf8e19993d54a27704fa416b78c71a1a7e4d2e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"b9ebabc3585ae300918cf374d0e27482685125100cecaafddede538fbb5a40a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"c625a214a177ed00125760f3e2405cc4232ca5f64fc78bc978d71fd75e9a6826","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"cbaacdba3cdb5f9615f3c30bd21831fd56c03f1d016fedcf5af22b163d2e6dbf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"df0551bed39aa73fe732d284c7e6d0ded1e61a2958e4c69ba2da428b19661330","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"eaaec004f93ac56e6a302a9b3b960e5dc8a3b2d0bac88e6134fbcb646be0164c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"f46eb4667284ae52474f9c5d1c56365279e44eb39c2013424595d6e31d0371dc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"0106b71417e491e529377ec2b45ed49d1465c0e633043fd4f30c3fa97c171f04","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"03a0dafe79333dbdc8c792d0760803f5086ef9cc7bbdd36ab7d3169b64c2e4d5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"0e791fb9346661a70c194cfb6e098fdc72aacc847837ef7aa5cd0658defd98d1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"14eb1554c2d31c5fae26b140930436e0f15586a4cdb3cfcfcbd958d9798ff26a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"14f9230915a46d82613307e0e88b555b3c64515b2ba7316cd91ba286907440a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"1655b8a33c3937e95689f115efa2e090ef3e795a250b746e22721f4b186ceb7d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"21f1e31067c5dd38ba01f887dce542146db81e2982c9a524c9ffbf3ab865ee13","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"22c20d09b59b89464c26d69757cea3917fd66fedb4ebbac1aa053bfdcc1f0958","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"536c39753e847972ea37a7b2f6ee10fe001d74b685e5f18a6b57e1e12d39aeb2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"58ab2df7f412acb10ced45b5797a7f346d05107c3b698548fb98f714075b10d5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"5cf5c4693d6056fe4f830217419a28853e24451b1aa973ae9682873a9a59669e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"62311dd5ee7648377e2f7c4d26440404e1a3a46c038b5f63c07809fc364fc6aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"6cd2adacceff2be8bbe5bcfbc6d22f3b1148b31733d56bee98ad0a0fbb39e77b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"6f45baea5fe95b50cc9556a7f22431899d9af180d73e093a60dd345fd2efb08e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"78fbd998914f2f0054e4ace8b13c8f9eac3cf7aef9f283f7ca59f5458dc6816e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"7b1567bbe32e5063f645c569e418e009c1e327f6ce830d15c806d901450143fe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"800c007a365cade784435141b097ab385e6c173214e3b870691e93bdea5b73a2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"80b7fb1d85d48245bc7bf717b61b3d95dadc758205d3da72ea9e4c94cc4a1f7a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"86ecd24df10021b7e780d3699d8b1856641d8c614dc0a419477be1ceee0f6eee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"95d9f89adf6dc46625bb4c3e80ea64838bb179a579b1669ee18c0248281f0d64","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"97b7668b955d925c4f43bb21f5348e9242dfb68ea801c33dbf722503a1b4ac02","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"9d6a29e936b9851a7d1323805a036deaa806b0785b4dbf52791e53490a33abed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"bd7af4c537c73358d02016ed70c772e25c8b072e5a199b9086fa7de4c8851a7c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"c0a5f3d8a8f7c09055b29769bddf9ef3a30bcd3286c1e4afaec31afe0653663a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"c1f0352b8ba95b2e7d2c6716919044b50be1c7163316d9882dec01ca3210179c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"c33ad9bc4ffde1ce4b4bd03d38fa9894428e92add1a529ec3802c3cea9bbf87c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"d2eebed913de5a27f1c4fd138d203441ac3d7be899cfbe2dd7fa1979aef39d76","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"d3f2f1eac10aeecd865f2f2f126ba031e5cca5a98fdd53b87a1bd214dfdae08f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"dcfb3f3dba165b20ab823ad1f5de19e9b194065ef9a9c040932e907d3c2a19f0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"ea3b6eeead8cb8a831605a65c6fffef9a96d277ed391217c61fa6050643cef12","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"ed2d1e41cb76d25bb880e5c2ed4abcb8a1ed7407c9126ba945b8a0fd92b42b91","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"f2ec8c7a15ca42900d5faa912291b6e6c0282652bfecf73385613a6b7215c028","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f7692883b7d8edf2d0661368bcbf0596b25bb01a118dcf71cc62820145611635","failure_pattern":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set: assertion failed: expected values to equal","search_query":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set: assertion failed: expected values to equal","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":70,"anchor_occurrences":7,"after_last_push_count":15,"trend_counts":[3,3,1,0,0,1,7],"trend_range":"2026-04-23..2026-04-29 UTC","prior_windows_present":1,"prior_window_starts":["2026-04-20"],"prior_runs_affected":15,"prior_last_seen_at":"2026-04-25T01:00:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set","occurrences":6},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/mise_routing.go:89]: Expected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/mise_routing.go:89]: Expected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049610459375472640","occurred_at":"2026-04-29T19:06:15Z","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049565492720439296","occurred_at":"2026-04-29T19:04:27Z","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049564091978092544","occurred_at":"2026-04-29T14:52:53Z","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049530806933655552","occurred_at":"2026-04-29T14:45:29Z","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049498421609369600","occurred_at":"2026-04-29T14:37:56Z","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":7,"runs_affected":2,"failed_runs":2,"run_impact_percent":20,"anchor_occurrences":7,"after_last_push_count":74,"trend_counts":[0,3,0,2,0,1,7],"trend_range":"2026-04-23..2026-04-29 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-30","2026-04-06","2026-04-20"],"prior_runs_affected":29,"prior_last_seen_at":"2026-04-26T11:50:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["hift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc000f45fe0 sp=0xc000f45f88 pc=0x3232cdb\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000f45fe8 sp=0xc000f45fe0 pc=0x2dc5cc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 103\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\ngoroutine 1020 gp=0xc0016121c0 m=nil [select]:\nruntime.gopark(0xc00008ba18?, 0x2?, 0xf0?, 0x7b?, 0xc00008ba0c?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc00008b898 sp=0xc00008b878 pc=0x2dbd50e\nruntime.selectgo(0xc00008ba18, 0xc00008ba08, 0xc00158a100?, 0x0, 0x3584d1c?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc00008b9d8 sp=0xc00008b898 pc=0x2d982f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8e08fe8, 0xc000937680}, 0x82de501?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc00008ba48 sp=0xc00008b9d8 pc=0x3584476\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8e57a80, {0x8e08fe8, 0xc000937680}, 0xc001638c90)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc00008bd48 sp=0xc00008ba48 pc=0x3c74152\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster({0x8e08fe8, 0xc000937630}, 0xc0012eb080, {0xc0008b9908, 0x16}, {0xc001502840, 0x12}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:198 +0x52a fp=0xc00008bef0 sp=0xc00008bd48 pc=0x6921b4a\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:467 +0x94 fp=0xc00008bf78 sp=0xc00008bef0 pc=0x6925134\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc00008bfe0 sp=0xc00008bf78 pc=0x463f270\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc00008bfe8 sp=0xc00008bfe0 pc=0x2dc5cc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1028\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 1028 gp=0xc0012c2540 m=nil [sync.WaitGroup.Wait, 22 minutes]:\nruntime.gopark(0xc00164ba80?, 0x2d54874?, 0x20?, 0x30?, 0x7f6d34649280?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000f4ba20 sp=0xc000f4ba00 pc=0x2dbd50e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:466\nruntime.semacquire1(0xc000c19390, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x21d fp=0xc000f4ba88 sp=0xc000f4ba20 pc=0x2d9959d\nsync.runtime_SemacquireWaitGroup(0xc00151cfa8?, 0x40?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0xc000f4bac0 sp=0xc000f4ba88 pc=0x2dbec2e\nsync.(*WaitGroup).Wait(0xc000c19388)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0xc000f4bae8 sp=0xc000f4bac0 pc=0x2de2225\ngolang.org/x/sync/errgroup.(*Group).Wait(0xc000c19380)\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:56 +0x1e fp=0xc000f4bb08 sp=0xc000f4bae8 pc=0x463f11e\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters({0x7f6d346da060, 0xc0014c55c0}, 0xc0012eb080, {0xc0008b9908, 0x16}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:470 +0x8ef fp=0xc000f4bdb0 sp=0xc000f4bb08 pc=0x6924aef\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).cleanupResourceGroup(0xc000cc2280, {0x7f6d346da060, 0xc0014c55c0}, {0xc0008b9908, 0x16}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:488 +0x29d fp=0xc000f4bf28 sp=0xc000f4bdb0 pc=0x694491d\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.func1({0x7f6d346da060, 0xc0014c55c0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:316 +0x14d fp=0xc000f4bfc0 sp=0xc000f4bf28 pc=0x6942ccd\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:325 +0x28 fp=0xc000f4bfe0 sp=0xc000f4bfc0 pc=0x6942b48\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000f4bfe8 sp=0xc000f4bfe0 pc=0x2dc5cc1\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 1004\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309 +0xc5\n\nrax    0xca\nrbx    0x0\nrcx    0x2dc7ae3\nrdx    0x0\nrdi    0xc2d5490\nrsi    0x80\nrbp    0x7ffc33e62900\nrsp    0x7ffc33e628b8\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0xc000090008\nr13    0x1\nr14    0xc2d33a0\nr15    0x5\nrip    0x2dc7ae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command\nhift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc000f45fe0 sp=0xc000f45f88 pc=0x3232cdb\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000f45fe8 sp=0xc000f45fe0 pc=0x2dc5cc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 103\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\ngoroutine 1020 gp=0xc0016121c0 m=nil [select]:\nruntime.gopark(0xc00008ba18?, 0x2?, 0xf0?, 0x7b?, 0xc00008ba0c?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc00008b898 sp=0xc00008b878 pc=0x2dbd50e\nruntime.selectgo(0xc00008ba18, 0xc00008ba08, 0xc00158a100?, 0x0, 0x3584d1c?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc00008b9d8 sp=0xc00008b898 pc=0x2d982f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8e08fe8, 0xc000937680}, 0x82de501?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc00008ba48 sp=0xc00008b9d8 pc=0x3584476\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8e57a80, {0x8e08fe8, 0xc000937680}, 0xc001638c90)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc00008bd48 sp=0xc00008ba48 pc=0x3c74152\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster({0x8e08fe8, 0xc000937630}, 0xc0012eb080, {0xc0008b9908, 0x16}, {0xc001502840, 0x12}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:198 +0x52a fp=0xc00008bef0 sp=0xc00008bd48 pc=0x6921b4a\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:467 +0x94 fp=0xc00008bf78 sp=0xc00008bef0 pc=0x6925134\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc00008bfe0 sp=0xc00008bf78 pc=0x463f270\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc00008bfe8 sp=0xc00008bfe0 pc=0x2dc5cc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1028\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 1028 gp=0xc0012c2540 m=nil [sync.WaitGroup.Wait, 22 minutes]:\nruntime.gopark(0xc00164ba80?, 0x2d54874?, 0x20?, 0x30?, 0x7f6d34649280?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000f4ba20 sp=0xc000f4ba00 pc=0x2dbd50e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:466\nruntime.semacquire1(0xc000c19390, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x21d fp=0xc000f4ba88 sp=0xc000f4ba20 pc=0x2d9959d\nsync.runtime_SemacquireWaitGroup(0xc00151cfa8?, 0x40?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0xc000f4bac0 sp=0xc000f4ba88 pc=0x2dbec2e\nsync.(*WaitGroup).Wait(0xc000c19388)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0xc000f4bae8 sp=0xc000f4bac0 pc=0x2de2225\ngolang.org/x/sync/errgroup.(*Group).Wait(0xc000c19380)\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:56 +0x1e fp=0xc000f4bb08 sp=0xc000f4bae8 pc=0x463f11e\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters({0x7f6d346da060, 0xc0014c55c0}, 0xc0012eb080, {0xc0008b9908, 0x16}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:470 +0x8ef fp=0xc000f4bdb0 sp=0xc000f4bb08 pc=0x6924aef\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).cleanupResourceGroup(0xc000cc2280, {0x7f6d346da060, 0xc0014c55c0}, {0xc0008b9908, 0x16}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:488 +0x29d fp=0xc000f4bf28 sp=0xc000f4bdb0 pc=0x694491d\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.func1({0x7f6d346da060, 0xc0014c55c0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:316 +0x14d fp=0xc000f4bfc0 sp=0xc000f4bf28 pc=0x6942ccd\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:325 +0x28 fp=0xc000f4bfe0 sp=0xc000f4bfc0 pc=0x6942b48\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000f4bfe8 sp=0xc000f4bfe0 pc=0x2dc5cc1\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 1004\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309 +0xc5\n\nrax    0xca\nrbx    0x0\nrcx    0x2dc7ae3\nrdx    0x0\nrdi    0xc2d5490\nrsi    0x80\nrbp    0x7ffc33e62900\nrsp    0x7ffc33e628b8\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0xc000090008\nr13    0x1\nr14    0xc2d33a0\nr15    0x5\nrip    0x2dc7ae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command","Running Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1777462557 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, Stage-And-Prod-Only]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:211\n  \"ts\"=\"2026-04-29 11:35:57.841702\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-be-able-to-forward-control-plane-logs-to-a-storage-account-and-Event-Hub-via-shoebox-diagnostic-settings|pid:959\"\n  \"ts\"=\"2026-04-29 11:35:57.851970\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating a resource group @ 04/29/26 11:35:57.852\n  \"ts\"=\"2026-04-29 11:35:57.852028\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"shoebox-cluster-g5nzsw\"\n    STEP: creating cluster parameters @ 04/29/26 11:36:01.176\n    STEP: creating customer resources @ 04/29/26 11:36:01.318\n    STEP: creating the HCP cluster @ 04/29/26 11:36:50.619\n  \"ts\"=\"2026-04-29 11:36:50.620015\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"shoebox-hcp-cluster\" \"resourceGroup\"=\"shoebox-cluster-g5nzsw\" \"version\"=\"4.20.19\" \"channelGroup\"=\"candidate\"\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:256 @ 04/29/26 12:21:50.621\n  \"ts\"=\"2026-04-29 12:21:50.621598\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-29 12:21:50.621671\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-hppzv9\" \"resourceGroup\"=\"shoebox-cluster-g5nzsw\"\n  \"ts\"=\"2026-04-29 12:21:50.744605\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-shoebox-hcp-cluster-hppzv9\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-29 12:21:52.538469\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-47f28969e60a59221ebcccd8b80e1b7f7b6f125a7ef279fc1cbfbef40d9caf18.yaml.gz\"\n  \"ts\"=\"2026-04-29 12:21:52.538617\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-47f28969e60a59221ebcccd8b80e1b7f7b6f125a7ef279fc1cbfbef40d9caf18.yaml.gz\"\n  \"ts\"=\"2026-04-29 12:21:52.538679\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-29 12:21:52.539890\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-146\"\n  \"ts\"=\"2026-04-29 12:21:52.539901\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"shoebox-cluster-g5nzsw\"\n  \"ts\"=\"2026-04-29 12:21:52.836851\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-146\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-29 12:21:52.947662\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-146\" \"deployment\"=\"msi-deployment-146\"\n  \"ts\"=\"2026-04-29 12:21:53.135379\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-g5nzsw\" \"deployment\"=\"pooledNonMsiScopedAssignments-22eudp3sazqeq\"\n  \"ts\"=\"2026-04-29 12:21:53.241656\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-g5nzsw\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-hppzv9\"\n  \"ts\"=\"2026-04-29 12:21:53.365413\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-29 12:21:53.636631\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-29 12:21:53.636773\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"shoebox-cluster-g5nzsw\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1777462557 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, Stage-And-Prod-Only]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:211\n  \"ts\"=\"2026-04-29 11:35:57.841702\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-be-able-to-forward-control-plane-logs-to-a-storage-account-and-Event-Hub-via-shoebox-diagnostic-settings|pid:959\"\n  \"ts\"=\"2026-04-29 11:35:57.851970\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating a resource group @ 04/29/26 11:35:57.852\n  \"ts\"=\"2026-04-29 11:35:57.852028\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"shoebox-cluster-g5nzsw\"\n    STEP: creating cluster parameters @ 04/29/26 11:36:01.176\n    STEP: creating customer resources @ 04/29/26 11:36:01.318\n    STEP: creating the HCP cluster @ 04/29/26 11:36:50.619\n  \"ts\"=\"2026-04-29 11:36:50.620015\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"shoebox-hcp-cluster\" \"resourceGroup\"=\"shoebox-cluster-g5nzsw\" \"version\"=\"4.20.19\" \"channelGroup\"=\"candidate\"\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:256 @ 04/29/26 12:21:50.621\n  \"ts\"=\"2026-04-29 12:21:50.621598\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-29 12:21:50.621671\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-hppzv9\" \"resourceGroup\"=\"shoebox-cluster-g5nzsw\"\n  \"ts\"=\"2026-04-29 12:21:50.744605\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-shoebox-hcp-cluster-hppzv9\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-29 12:21:52.538469\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-47f28969e60a59221ebcccd8b80e1b7f7b6f125a7ef279fc1cbfbef40d9caf18.yaml.gz\"\n  \"ts\"=\"2026-04-29 12:21:52.538617\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-47f28969e60a59221ebcccd8b80e1b7f7b6f125a7ef279fc1cbfbef40d9caf18.yaml.gz\"\n  \"ts\"=\"2026-04-29 12:21:52.538679\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-29 12:21:52.539890\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-146\"\n  \"ts\"=\"2026-04-29 12:21:52.539901\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"shoebox-cluster-g5nzsw\"\n  \"ts\"=\"2026-04-29 12:21:52.836851\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-146\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-29 12:21:52.947662\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-146\" \"deployment\"=\"msi-deployment-146\"\n  \"ts\"=\"2026-04-29 12:21:53.135379\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-g5nzsw\" \"deployment\"=\"pooledNonMsiScopedAssignments-22eudp3sazqeq\"\n  \"ts\"=\"2026-04-29 12:21:53.241656\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-g5nzsw\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-hppzv9\"\n  \"ts\"=\"2026-04-29 12:21:53.365413\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-29 12:21:53.636631\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-29 12:21:53.636773\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"shoebox-cluster-g5nzsw\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","b/golang/src/runtime/select.go:351 +0x8b7 fp=0xc00008b9d8 sp=0xc00008b898 pc=0x2d982f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8e08fe8, 0xc000630460}, 0x82de501?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc00008ba48 sp=0xc00008b9d8 pc=0x3584476\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8e57a80, {0x8e08fe8, 0xc000630460}, 0xc0015b55d0)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc00008bd48 sp=0xc00008ba48 pc=0x3c74152\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster({0x8e08fe8, 0xc000630410}, 0xc00158a4b0, {0xc000c00c60, 0x17}, {0xc0015b6550, 0xe}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:198 +0x52a fp=0xc00008bef0 sp=0xc00008bd48 pc=0x6921b4a\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:467 +0x94 fp=0xc00008bf78 sp=0xc00008bef0 pc=0x6925134\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc00008bfe0 sp=0xc00008bf78 pc=0x463f270\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc00008bfe8 sp=0xc00008bfe0 pc=0x2dc5cc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1110\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 1065 gp=0xc00172e540 m=nil [sync.WaitGroup.Wait, 27 minutes]:\nruntime.gopark(0xc0015535f0?, 0x2d54874?, 0x80?, 0x41?, 0x200c00156e1c0?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000c13590 sp=0xc000c13570 pc=0x2dbd50e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:466\nruntime.semacquire1(0xc00151fdb8, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x21d fp=0xc000c135f8 sp=0xc000c13590 pc=0x2d9959d\nsync.runtime_SemacquireWaitGroup(0xc00051f7c0?, 0x40?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0xc000c13630 sp=0xc000c135f8 pc=0x2dbec2e\nsync.(*WaitGroup).Wait(0xc00151fdb0)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0xc000c13658 sp=0xc000c13630 pc=0x2de2225\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups(0xc000f67a40, {0x7f4ed82dc460, 0xc00146cfc0}, {{0xc000531300, 0x1, 0x1}, 0x34630b8a000, {0x82c80f2, 0x8}})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:327 +0x273 fp=0xc000c13768 sp=0xc000c13658 pc=0x6942793\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources(0xc000f67a40, {0x7f4ed82dc460, 0xc00146cfc0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:228 +0x225 fp=0xc000c13888 sp=0xc000c13768 pc=0x6941f25\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources-fm({0x7f4ed82dc460?, 0xc00146cfc0?})\n\t\u003cautogenerated\u003e:1 +0x33 fp=0xc000c138b0 sp=0xc000c13888 pc=0x6956053\nruntime.call16(0xc0016a0870, 0xc0006b4000, 0x0, 0x0, 0x0, 0x10, 0xc000c13b08)\n\t/usr/lib/golang/src/runtime/asm_amd64.s:774 +0x43 fp=0xc000c138d0 sp=0xc000c138b0 pc=0x2dc4223\nruntime.reflectcall(0x7cbdf00?, 0xc00146cfc0?, 0x2?, 0x82fdae4?, 0x0?, 0x12?, 0x78a0d80?)\n\t\u003cautogenerated\u003e:1 +0x36 fp=0xc000c13910 sp=0xc000c138d0 pc=0x2dccab6\nreflect.Value.call({0x7181340?, 0xc0006b4000?, 0xc0014a3650?}, {0x82bbf7b, 0x4}, {0xc000317f60, 0x1, 0x0?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcc6 fp=0xc000c13e70 sp=0xc000c13910 pc=0x2dffa86\nreflect.Value.Call({0x7181340?, 0xc0006b4000?, 0xc0014a3650?}, {0xc000317f60?, 0x5?, 0xc00149a4b0?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9 fp=0xc000c13ee8 sp=0xc000c13e70 pc=0x2dfeb79\ngithub.com/onsi/ginkgo/v2/internal.NewCleanupNode.func2({0x8e21b60?, 0xc00146cfc0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:589 +0x150 fp=0xc000c13f88 sp=0xc000c13ee8 pc=0x3220210\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc000c13fe0 sp=0xc000c13f88 pc=0x3232cdb\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000c13fe8 sp=0xc000c13fe0 pc=0x2dc5cc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 104\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nrax    0xfffffffffffffffc\nrbx    0x0\nrcx    0x2dc7ae3\nrdx    0x0\nrdi    0xc2d75d0\nrsi    0x80\nrbp    0x7f4edc420cb8\nrsp    0x7f4edc420c70\nr8     0x0\nr9     0x0\nr10    0x7f4edc420ca8\nr11    0x206\nr12    0x7f4edc420ca8\nr13    0x1\nr14    0xc000002c40\nr15    0x0\nrip    0x2dc7ae3\nrflags 0x206\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command\nb/golang/src/runtime/select.go:351 +0x8b7 fp=0xc00008b9d8 sp=0xc00008b898 pc=0x2d982f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8e08fe8, 0xc000630460}, 0x82de501?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc00008ba48 sp=0xc00008b9d8 pc=0x3584476\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8e57a80, {0x8e08fe8, 0xc000630460}, 0xc0015b55d0)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc00008bd48 sp=0xc00008ba48 pc=0x3c74152\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster({0x8e08fe8, 0xc000630410}, 0xc00158a4b0, {0xc000c00c60, 0x17}, {0xc0015b6550, 0xe}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:198 +0x52a fp=0xc00008bef0 sp=0xc00008bd48 pc=0x6921b4a\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:467 +0x94 fp=0xc00008bf78 sp=0xc00008bef0 pc=0x6925134\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc00008bfe0 sp=0xc00008bf78 pc=0x463f270\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc00008bfe8 sp=0xc00008bfe0 pc=0x2dc5cc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1110\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 1065 gp=0xc00172e540 m=nil [sync.WaitGroup.Wait, 27 minutes]:\nruntime.gopark(0xc0015535f0?, 0x2d54874?, 0x80?, 0x41?, 0x200c00156e1c0?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000c13590 sp=0xc000c13570 pc=0x2dbd50e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:466\nruntime.semacquire1(0xc00151fdb8, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x21d fp=0xc000c135f8 sp=0xc000c13590 pc=0x2d9959d\nsync.runtime_SemacquireWaitGroup(0xc00051f7c0?, 0x40?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0xc000c13630 sp=0xc000c135f8 pc=0x2dbec2e\nsync.(*WaitGroup).Wait(0xc00151fdb0)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0xc000c13658 sp=0xc000c13630 pc=0x2de2225\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups(0xc000f67a40, {0x7f4ed82dc460, 0xc00146cfc0}, {{0xc000531300, 0x1, 0x1}, 0x34630b8a000, {0x82c80f2, 0x8}})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:327 +0x273 fp=0xc000c13768 sp=0xc000c13658 pc=0x6942793\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources(0xc000f67a40, {0x7f4ed82dc460, 0xc00146cfc0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:228 +0x225 fp=0xc000c13888 sp=0xc000c13768 pc=0x6941f25\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources-fm({0x7f4ed82dc460?, 0xc00146cfc0?})\n\t\u003cautogenerated\u003e:1 +0x33 fp=0xc000c138b0 sp=0xc000c13888 pc=0x6956053\nruntime.call16(0xc0016a0870, 0xc0006b4000, 0x0, 0x0, 0x0, 0x10, 0xc000c13b08)\n\t/usr/lib/golang/src/runtime/asm_amd64.s:774 +0x43 fp=0xc000c138d0 sp=0xc000c138b0 pc=0x2dc4223\nruntime.reflectcall(0x7cbdf00?, 0xc00146cfc0?, 0x2?, 0x82fdae4?, 0x0?, 0x12?, 0x78a0d80?)\n\t\u003cautogenerated\u003e:1 +0x36 fp=0xc000c13910 sp=0xc000c138d0 pc=0x2dccab6\nreflect.Value.call({0x7181340?, 0xc0006b4000?, 0xc0014a3650?}, {0x82bbf7b, 0x4}, {0xc000317f60, 0x1, 0x0?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcc6 fp=0xc000c13e70 sp=0xc000c13910 pc=0x2dffa86\nreflect.Value.Call({0x7181340?, 0xc0006b4000?, 0xc0014a3650?}, {0xc000317f60?, 0x5?, 0xc00149a4b0?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9 fp=0xc000c13ee8 sp=0xc000c13e70 pc=0x2dfeb79\ngithub.com/onsi/ginkgo/v2/internal.NewCleanupNode.func2({0x8e21b60?, 0xc00146cfc0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:589 +0x150 fp=0xc000c13f88 sp=0xc000c13ee8 pc=0x3220210\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc000c13fe0 sp=0xc000c13f88 pc=0x3232cdb\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000c13fe8 sp=0xc000c13fe0 pc=0x2dc5cc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 104\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nrax    0xfffffffffffffffc\nrbx    0x0\nrcx    0x2dc7ae3\nrdx    0x0\nrdi    0xc2d75d0\nrsi    0x80\nrbp    0x7f4edc420cb8\nrsp    0x7f4edc420c70\nr8     0x0\nr9     0x0\nr10    0x7f4edc420ca8\nr11    0x206\nr12    0x7f4edc420ca8\nr13    0x1\nr14    0xc000002c40\nr15    0x0\nrip    0x2dc7ae3\nrflags 0x206\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"0969c6b48e157b4cfbe6215b24122736ff778076fa4db43068de8185be8091e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"6321a04cae48565708d281dae2b9878bccd895f47233ae416b75653b52327bd5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"84c2e8a74ba33c31f0be0f7198b10f697b056915254ecba14e745053e5125602","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"a3d5611aa9faef5d88aca3e367072cb6cb757edfc0f4fa7689439c927bf8c464","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"bc8661b370fce57389ec86e649c94658ee15664bf8f7de3af0949f6867c26751","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"12fe864a2d1bc714c80dc0d953007cc4efd85887bf3796616904dc81ac6f74b3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"9bbceb5dbc42c17445082b6aa90dae9e87301a0937b0d9a7e5cf4ce97d2c9c39","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e9c6c185c09aa6c23477c53c1d3d7fb1bb60e041c3cb718558ce145b6718d4d","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":20,"anchor_occurrences":2,"after_last_push_count":8,"trend_counts":[1,1,1,1,0,1,2],"trend_range":"2026-04-23..2026-04-29 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-30","2026-04-06","2026-04-20"],"prior_runs_affected":9,"prior_last_seen_at":"2026-04-26T11:50:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 2700.007s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 2700.007s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"0c4ed9bba8fe112ea7971d2241bf4fe4d2d0df6edae20700d459cff3d63baf26","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"f816448949a54c698daec440b6be14869199dd5156942a339f2ff5a445aaf853","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":20,"anchor_occurrences":2,"after_last_push_count":28,"trend_counts":[0,0,0,1,0,0,2],"trend_range":"2026-04-23..2026-04-29 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-30","2026-04-06","2026-04-20"],"prior_runs_affected":37,"prior_last_seen_at":"2026-04-26T11:50:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: Interrupted by User","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:183]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:183]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"9626adf85231cae7571909c00084bdfb7c9de3df3208a3d5200ee95baf8fc7e8","failure_pattern":"context deadline exceeded","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":20,"anchor_occurrences":2,"after_last_push_count":6,"trend_counts":[1,0,1,1,0,1,2],"trend_range":"2026-04-23..2026-04-29 UTC","prior_windows_present":1,"prior_window_starts":["2026-04-20"],"prior_runs_affected":6,"prior_last_seen_at":"2026-04-26T11:50:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: Unexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: Unexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"b15c55d32190fb3d32f16f58a52f080055a52cddf394a6cd7e062c8467f788e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2049307708154187776","occurred_at":"2026-04-29T02:00:06Z","signature_id":"b15c55d32190fb3d32f16f58a52f080055a52cddf394a6cd7e062c8467f788e4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"443d5bf877a5a8072106dcd7ee154f7ec5cf70e7883b77cc29306405e54f0131","failure_pattern":"Process did not finish before 4h0m0s timeout","search_query":"Process did not finish before 4h0m0s timeout","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":10,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,2,0,0,1],"trend_range":"2026-04-23..2026-04-29 UTC","prior_windows_present":1,"prior_window_starts":["2026-04-20"],"prior_runs_affected":3,"prior_last_seen_at":"2026-04-26T22:36:45Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":1}],"full_error_samples":["{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 4h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-29T13:05:55Z\"}"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049411223392358400","occurred_at":"2026-04-29T08:51:26Z","signature_id":"e866a7b39ddf32d1bf58138de6de78cb4986cc8b8b5bb67431025636d69fa4bb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"63a755b10a4ebc8c89991a40e36c8d3c0a9cd5e114a858de40038cb6cec08053","failure_pattern":"add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API","search_query":"add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":10,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-04-23..2026-04-29 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:252]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e3f710\u003e: \n    add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:252]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e3f710\u003e: \n    add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2049498421609369600","occurred_at":"2026-04-29T14:37:56Z","signature_id":"1595ba64259ffd3f315c00795574162bbc9ddf8f6eb854b0637d888588a0f4ed","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}