{"meta":{"start_date":"2026-05-11","end_date":"2026-05-17","start_at":"2026-05-11T00:00:00Z","end_at":"2026-05-18T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T11:03:05Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":32,"failed_runs":10,"raw_occurrences":132,"matched_occurrences":132,"runs_affected":10},"rows":[{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-52hx56, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-52hx56\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-52hx56, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":89,"runs_affected":6,"failed_runs":6,"run_impact_percent":18.75,"anchor_occurrences":89,"after_last_push_count":398,"trend_counts":[0,0,1,10,33,44,1,0],"trend_range":"2026-05-10..2026-05-17 UTC","prior_windows_present":4,"prior_window_starts":["2026-04-13","2026-04-20","2026-04-27","2026-05-04"],"prior_runs_affected":31,"prior_last_seen_at":"2026-05-08T17:54:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000bc92c0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-vvj79d, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-vvj79d\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-vvj79d, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000bc92c0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-vvj79d, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-vvj79d\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-vvj79d, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012da750\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-b24hg5, error: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-b24hg5\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-b24hg5, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012da750\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-b24hg5, error: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-b24hg5\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-b24hg5, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001198c00\u003e: \n    failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-8rrqnq, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-8rrqnq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-8rrqnq, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001198c00\u003e: \n    failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-8rrqnq, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-8rrqnq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-8rrqnq, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055452538798346240","occurred_at":"2026-05-16T00:57:28Z","signature_id":"267ac56f03dadd61e250f265d056040776c26e75d85b03e73b95262c3407a382","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"066f2dfa375388702a10c5857b6b7bb78752fe7ebe2c0e53de4d61a1f44d8976","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"0feea2784d3d05360db6d9da139f236df38b494475443383e2a356b5760dff04","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"120381f5e7db5682f04ef466e975d5e2835b84279c0bd88e1d2e97f9cffc23e7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"16e6f5e2aebcb495c11c5ce391dd9faeeeeb31701d820ff4e2b3711bf2d52ed1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"18432de74bd5821ce79664aecfef6420651271c73a67e86737b0769b73e8d188","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"2e921268814ae9e61428a3a3e3a51344692318200ae378f589e8de6844752b85","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"32e2856f36e26691956447685ba171883f24d94a09ca55670b31bf691b9566bc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"349e97a75a75c4492922810881185ee1b8c14f65760d9d90452231da12601a76","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"3f2ecb2726f342eaeb67d6272414959d9c873aa0d4191632542099fa528eaeab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"472930aaa28188f1ddc3960eb075b488ccf6cbc4e842f95a9002a05cca7860ab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"4d29133fe965a20093de9cefda82179b7516dd06f4df9b56d698b1ef1b6676f2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"675fae08b5ef9455dde21d6e0314cfa0943866d4ddfdda6e86e66edbcd4bed60","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"6a20024dac504976f5031963e5a111b09abe55552a5b10e11d9c1c725193419c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"6fb23152bda9ddb48b80726bb687ade0ab1328897b3135de8966b6e76cc11bda","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"72d987a43d46bb5557d3769a37a2701e2169f020f93925b0b5325873769bd0f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"799b300176dbdaf8ac2480fec8c01aa29e9ff7d3a3b7a52de8bc8666a7e62255","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"80592fe0eca8130018f8d2e353f9f90de9d5acea257fe7036b3c9e71fc043232","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"80ec19bb6c8fa88036014cae68184a4daadd3e7e10533b00e883526ff4a17a51","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"825707ea9cbc16a5fde49e4937b3ad39cbb0f9d47445b5aa0b46e3f36ca61a73","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"863c386843a9f274eac4d0063428316696ecbd7d95cd3a0fd1633557e53d4305","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"917703cf9a6e1aee9fae5ebd4ab9a3e6b3339b25975b57d770bc41ea2df6c411","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"97eb4f8e426803c4c9479c12d66e386355f26371afaf092d9a1b3030174e688b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"9b53cf9e82a060c4603fa2eee952cd1e8f284163fa08429e47e1f02977f8f6f6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"a928e3d89723bf62c7fb5077aa40537f8477bd0d815bed98f813bff81f52a013","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"ad22014269302414fb69d0a91ea8cb886d650454be5fd399657f0f307e3eea6c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"b05103efa7b10dec24519568bb598ca94e7bca0197b7233640a4d47fd9592972","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"b0d59a8a0b2b4bd44e30366c5567a48d537668629579035b9715d60d8d3bccaa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"ba0bde870254f82a01db91957251d0e9188cc39fe9882b9a5b30d5f9da088649","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"c6a6d2f74d5fb42091cceb9ac8cbfbd26c424eb05fce977ea750bc398704d889","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"d5ca7e7517151d31f585b30ae36d558707d5eefe402de34df716d7eec679a3cf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"e05e07b9f688fcb72ff2bba9b39ace7a45386c8e16fa46bf20fcddbf601fe338","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"e2ffe336168990b5c4520e9e044a1c46d96c9639a86a4b23aedb418f3a0623a9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"eb520c3b399528f1230c1631de5a23a84d701b3cd3b1d262d9c9bc596c90cf77","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"f3ac13e3923587b5782c2e114f5d0bf90a3c7102a239ad1f2891a82b99e9e101","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"ffb7e72404b26f1e3f01c77372192c6ff69fab75a2f49e5f9943206b12993611","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055186423362883584","occurred_at":"2026-05-15T07:20:01Z","signature_id":"0d93243dbd89897c6c4da7fb83cc4578735c59ed52fe944551db4a1e40cd4cc0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055186423362883584","occurred_at":"2026-05-15T07:20:01Z","signature_id":"2f471bd65a715bea46ddf77d1b4395e4e0397a2996e3bd5fc3913a2a456a94ee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055186423362883584","occurred_at":"2026-05-15T07:20:01Z","signature_id":"381a835728024b3c112004dcc7ae0a0da8178b158766700a1f32dea7f2675b87","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055186423362883584","occurred_at":"2026-05-15T07:20:01Z","signature_id":"6f9ed37348cdf2cd61052bdd03370bcde9cd417fa6ba17cf4c9912d3e4d494eb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055186423362883584","occurred_at":"2026-05-15T07:20:01Z","signature_id":"80d4bd88f2f9b39e419b12659954c31114989d14dcf2dd5e7662e8200a1a8f2c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055186423362883584","occurred_at":"2026-05-15T07:20:01Z","signature_id":"88cb6c8ec02ad6fb6c6fb0bca4c917e9fc48a12e5bc283dad54ce3eaf42a56b8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055186423362883584","occurred_at":"2026-05-15T07:20:01Z","signature_id":"b25d5f5dce6eb681441d31380330b9f883bc9522b683cda27a07d306bff217cb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055186423362883584","occurred_at":"2026-05-15T07:20:01Z","signature_id":"bd84a0b9bb5bba2699e2bd8c2c37d492858f0d6469fed7b8affe4b3c8a1e36f2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055186423362883584","occurred_at":"2026-05-15T07:20:01Z","signature_id":"e4e4a7031de1443b348b881998588b975d93afb24b337930c96a438220963290","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"06a942c253ba3ccfbd5503f29b68b5127ab91954e343add6706c6d833907d4e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"0d85d7d60eb7a10432e3bbdfd33a328e7c56f70c1c3b9ca5fb647f0838908d25","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"100030e6a462d0ee15ec9ac25b079bac65bd52a889462bd57812e74bbef4ed55","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"2b70180fe6e9a3e08736577d3364c5cef8b895822c8e23a547a0da56da96743b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"2eeb55b98e5ed80c3788ac690e982f2bc6594f74191474fecc0efdcb2f18dfb4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"3140118a2e1da9e226a23fe8f78a95a139a41ed84b4888c1085be5423ac50a13","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"45b6fed9008d8d4b945f64f49b1fd2448357456128c7507ee5940caabfa8b90f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"4b986ffcc471b2939206ceb91f887fe3c82d5fd752f3b4a5cce04aeca7c6c3e6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"4dce15946c522dc017005d62460ad9d54537dc891fa9ff06912bcd5218c8f37b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"4f6a45affe048b133ea59721600b51a00f12f8cc9d82cfe712af2335aec69f90","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"6005b2d98f68bc530d0cab9f9093f9856168a478daebc19776dd36ca9157904d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"64f5a76fdd56d2e82c88cf2ba94779d8bc1c2155042783d3cdbfc9cbff993840","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"6a673a972b8c5e6de9bcc86a1c3eb7efde553944650407885a38e18365ca3ba4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"6c1224ff2d2ca8ef2c05bd5e5aa379e13fec27716d734cac2332262d858d7824","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"6fd6a0d82e6ac81a8a51781f46ff4d8ff4c3a64bb765a4493f1d80fcbb8a25df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"72d9f854419130a82c3db29ba614039aef214eec9a2cf400c951677b0ae08c0c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"792a25e2e10b0a34f58c4e0b4144df6d7c52f1607e0a4b72579f602b9ee49da4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"7c78f6312b1140909309c1237ff954b51744c16f2526e88b209641ffcd9e8e52","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"8449631155d7574b04392b0bd24344c99399f14c518460e7802450e5796ba274","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"879e027658405d74dd42b34c281a3f801a2dc94beb7559ea47590750fc5bc65d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"8a9c42925de27475a278dd941d0955539e74316a33d6239db15406237f761192","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"8bbcbe7bac90cccc13d0c43fcca904077d68c651057fec67029be5054a4d188d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"adb27fab16992e0383eec5285b19fb40e7cd6daa662149242743ebf18ed5b63f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"b6761bfd06c6a0071a8d9e95c60bc9843ca7ca9889d98dbf24fc8681936c3b09","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"ba8cd21ec1d64252292a42117ca139fcb02e98e68c59759c94832f0f2a282967","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"bbedbb39eff251c15aee469f32cffc7f7b8935329ce9663673e581cd16bef01d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"be3e8f84d59dfa0dfd6d92a2378a88a0274af877024cf0bf939555b2bdbfe3f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"d4a5b7dd913ef11c91c9ea3fd502e59458ad64e77608d4b9c324ac341d431aef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"d96038c69e21cdff6c8509e8aebd3686a93a2a34f020674421eb1ddef0b9dd61","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"dab0d37553b0d564674d56ee36b1d4a41ca9239b73ceb5e9bc47b5e052198513","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"e264c01b515076aad759051f1d1f3e7944a60135f2e535ba495bd0c982c93c49","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"ed829806a7443cea416802ee1f6b0f511059ca32563cab594528efad81984471","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"fb96aaafd46c27d74bf48b43fa07a4f52bea68d241ae3e63cb3a9ed0a4830e8d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"030923b4469f49c276ffa3b9d92dd0cf199e43d610be3cffa33cc36d47d7dcfc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"259c0c4878a3294064ff783427a9495335289af4a76e2c2969548d6d7488f93b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"3dfcc3db6e67421d715c86030d8c6f40321e1e0e9b5bcccb610633797a87a7fe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"44d55884625a502599ab8997cb52ee4ea61cf80a6abe3811fc1118481f9c1609","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"5357482ff042392fbea7d971b3d746379a08250e4eda2d00cc87e2afdf6179d7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"64d6c6f613a99bfd4ac4c8fc12ba47f3a00c3ee4f88934fd14baba0eaf3935e3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"690ef8ee25232a11d83d13e985fc55e2beae046272b8bf29cf0835a919334a1d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"6e54c815df2350b1a97a61536eea7c7b1ab0538e2f3ee5777dae47741175f3f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"b33e3f53b73db25f34c0bf8198361fdc365adc906107819227046e2433841085","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"e17c66a32ce4618afc697d2fbd63321cbed74d3295fef46f766daf20d76c35ce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054040834738753536","occurred_at":"2026-05-12T03:27:51Z","signature_id":"c8db038de9b40b171765180a03cfe402b6b925a06aae27df84549bd25dd43d79","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":26,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.125,"anchor_occurrences":26,"after_last_push_count":28,"trend_counts":[0,0,0,26,0,0,0,0],"trend_range":"2026-05-10..2026-05-17 UTC","prior_windows_present":2,"prior_window_starts":["2026-04-13","2026-04-20"],"prior_runs_affected":4,"prior_last_seen_at":"2026-04-24T04:34:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:249]: Unexpected error:\n    \u003c*errors.joinError | 0xc000914120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-l8jbbf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-21-7mdv5l\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:249]: Unexpected error:\n    \u003c*errors.joinError | 0xc000914120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-l8jbbf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-21-7mdv5l\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:249]: Unexpected error:\n    \u003c*errors.joinError | 0xc002316300\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-9t7gt7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:249]: Unexpected error:\n    \u003c*errors.joinError | 0xc002316300\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-9t7gt7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:249]: Unexpected error:\n    \u003c*errors.joinError | 0xc0016deae0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-8d74qd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:249]: Unexpected error:\n    \u003c*errors.joinError | 0xc0016deae0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-8d74qd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"0b584415d65ca756dc1f852cacb82c8b1cb0771ff14ce6aca794848b692dfac0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"0e092dfc089508238761c8154527c066ca093c794f78ec3658e5fc62fda41fca","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"1a523784fc17094432cbebea9ef3c3990e7e65e3160d39cce27b5493c26bc7b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"1b01a2358b1324e52d86774e3d09fe12e8ce57538b93849d5b14c5dad8c3a99f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"3eb46e2b8d29c75fdd9d5600533633868ab21b6781653b095cd0d00e18845629","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"4343bda930af1d4d130f3c5d1c4af7157200b428137822b6a64dfa2a7da41f72","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"4774e96c1f08da60fecbee19ef267c6e5a6595db2258a389c16c4d71595be2f2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"4d86dfb4ce7427564a25d2034c682684abb26a6dd157e0fc1c1f78ec678b1d49","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"5827049d7e6492868366d0ae5760056ccfd29d36da41ccdf8f261bfc2278eef0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"721ebfd6b67998009a5c051c5d25b511a9a7fa5726da760dfe593b5f203799c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"82422424c94ef75bd336c1582883293b59b03a18b76921e85ee996e9a093936a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"897b6d42d29508a7e0d75d0b6f591e563ec2f30e6d9aadcbb3b5f18e4bec14df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"9d0071c8e13276d0a4d84c28e0e1b88a94149f5fe6bb6ee9d7e3f52e00851d78","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"a6747560da9449a2f2e81ea68acab503d2a38cf0a9cf363fd67b9e2648a200ed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"a9f1f812400bbd6039386e30f1d8dc0fc8758e0c34b0d4a8cecb2976037597c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"ad6c83aca6f0f44b891dee02bf7acd3bbd411f983fe4f3d3ae9306faf8730375","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"b9460cc801d5b768238c66284013cff0e867ddc85d18f6e23740f062d5727ea2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"c4290dab4432159dc398b66d53c867be46f0b74fa3ba45433cfd912ec410f87a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"c7fb262dbbb788f27970a30428d1e85a3513a9aa4a15836e86ce1003b6a671e7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"e2b446673f23a21f630814838e42b7f2fe822538c3cf8a7407c0e8228b205de0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"e54de82749ecdadc090c41d9e6440642e72bcdd5f0b12ee5381bb7c56e3df56f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"f6d978c9b54069debe7f2eec58c07f327faa76854adf97b4eddf940ee904a7a5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"f961b814cd69f893234b662be65bfaab8ee0272eadc6af27dbed6c78a917d8f8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"faf66d2d6fc1c695df517a3cc97b3a83a23af9a04ebb049f53116ebf5c273ed8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"fb3c65762199e5caee5359245295ccefb0593299a8ce0b0dfca21d26d9112172","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"ff8b606eafc646464a00e9ea418a489caef8707475870d411997392c1ea89c4f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"9626adf85231cae7571909c00084bdfb7c9de3df3208a3d5200ee95baf8fc7e8","failure_pattern":"context deadline exceeded","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":9.375,"anchor_occurrences":3,"after_last_push_count":14,"trend_counts":[0,0,0,0,1,2,0,0],"trend_range":"2026-05-10..2026-05-17 UTC","prior_windows_present":3,"prior_window_starts":["2026-04-20","2026-04-27","2026-05-04"],"prior_runs_affected":14,"prior_last_seen_at":"2026-05-08T09:23:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:132]: Unexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:132]: Unexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"b15c55d32190fb3d32f16f58a52f080055a52cddf394a6cd7e062c8467f788e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055186423362883584","occurred_at":"2026-05-15T07:20:01Z","signature_id":"b15c55d32190fb3d32f16f58a52f080055a52cddf394a6cd7e062c8467f788e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"b15c55d32190fb3d32f16f58a52f080055a52cddf394a6cd7e062c8467f788e4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":9.375,"anchor_occurrences":3,"after_last_push_count":4,"trend_counts":[0,0,1,1,0,0,1,0],"trend_range":"2026-05-10..2026-05-17 UTC","prior_windows_present":1,"prior_window_starts":["2026-04-20"],"prior_runs_affected":4,"prior_last_seen_at":"2026-04-24T18:17:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00045f2a0\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-ssw7m.apps.aro.i8p5a8p3n4h1h0k.ioif.brazilsouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00045f2a0\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-ssw7m.apps.aro.i8p5a8p3n4h1h0k.ioif.brazilsouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:173]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0005ab2e0\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-ct2x6.apps.aro.y8e3k0c7u4p5b7t.h4i0.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:173]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0005ab2e0\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-ct2x6.apps.aro.y8e3k0c7u4p5b7t.h4i0.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:168]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000b6ea80\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-46kk6.apps.aro.cilium-cl.5vr5.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:168]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000b6ea80\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-46kk6.apps.aro.cilium-cl.5vr5.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055564671175364608","occurred_at":"2026-05-16T08:23:00Z","signature_id":"b692bc5be2e476ea0acbdb8eb931ecb7c09469fc70870bd8f72b0dcf29f8900a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2054382242708328448","occurred_at":"2026-05-13T02:00:06Z","signature_id":"579d418056e7a8f42a03faa9816418f84d18ed55d09b4b4a5e792a175c4e6291","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054103528137297920","occurred_at":"2026-05-12T07:36:59Z","signature_id":"c5c813b1bea6876a30b6d427a25761ffe22d9710dabebbceceff87e73e0e84cd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.125,"anchor_occurrences":3,"after_last_push_count":22,"trend_counts":[0,0,0,0,3,0,0,0],"trend_range":"2026-05-10..2026-05-17 UTC","prior_windows_present":3,"prior_window_starts":["2026-04-20","2026-04-27","2026-05-04"],"prior_runs_affected":8,"prior_last_seen_at":"2026-05-04T06:07:27Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1}],"full_error_samples":["Customer should be able to create an HCP cluster and custom node pool osDisk size (Spec Runtime: 2h29m59.889s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:35\n        In [It] (Node Runtime: 2h29m59.889s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:35\n          At [By Step] creating the HCP cluster (Step Runtime: 13m58.338s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:73\n\n          Spec Goroutine\n          goroutine 76 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8e29e48, 0xc0013e8690}, 0x82faaf2?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8e66d20, {0x8e29e48, 0xc0013e8690}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x8e29eb8?, 0xc0014043f0?}, {{0x8e42958?, 0xc000d9c600?}, 0xc000798b30?}, 0xc00008f760, {0xc000080990, 0x13}, {0x8319ec1, 0x12}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:852\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0xc0013f6c97, 0x7}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:346\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func9.2({0x7fdc50f03d38, 0xc000baa1e0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:74\n                | \n                | By(\"creating the HCP cluster\")\n                \u003e err = tc.CreateHCPClusterFromParam(ctx,\n                | \tGinkgoLogr,\n                | \t*resourceGroup.Name,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8e42a30?, 0xc000baa1e0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 96\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80 @ 05/14/26 05:30:00.526\n  \"ts\"=\"2026-05-14 05:30:00.526801\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-05-14 05:30:00.526855\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-hcp-cluster-np-128-xgjwjl\" \"resourceGroup\"=\"clusternp128-dw5kpk\"\n  \"ts\"=\"2026-05-14 05:30:00.645848\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-hcp-cluster-np-128-xgjwjl\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-05-14 05:30:00.988854\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-a85d76e69c62fcc4973899bcc5957cf5e02900c7383d8dbab637c1fde0fca561.yaml.gz\"\n  \"ts\"=\"2026-05-14 05:30:00.988958\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-a85d76e69c62fcc4973899bcc5957cf5e02900c7383d8dbab637c1fde0fca561.yaml.gz\"\n  \"ts\"=\"2026-05-14 05:30:00.989012\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-05-14 05:30:00.991791\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-16\"\n  \"ts\"=\"2026-05-14 05:30:00.991804\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"clusternp128-dw5kpk\"\n  \"ts\"=\"2026-05-14 05:30:01.307927\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"clusternp128-dw5kpk\" \"deployment\"=\"pooledNonMsiScopedAssignments-3ljyjjx2n7gxo\"\n  \"ts\"=\"2026-05-14 05:30:01.324197\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-16\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-05-14 05:30:01.368575\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"clusternp128-dw5kpk\" \"deployment\"=\"customer-infra-hcp-cluster-np-128-xgjwjl\"\n  \"ts\"=\"2026-05-14 05:30:01.445985\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-16\" \"deployment\"=\"msi-deployment-16\"\n  \"ts\"=\"2026-05-14 05:30:01.570429\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-05-14 05:30:01.771943\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-05-14 05:30:01.772204\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"clusternp128-dw5kpk\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nCustomer should be able to create an HCP cluster and custom node pool osDisk size (Spec Runtime: 2h29m59.889s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:35\n        In [It] (Node Runtime: 2h29m59.889s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:35\n          At [By Step] creating the HCP cluster (Step Runtime: 13m58.338s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:73\n\n          Spec Goroutine\n          goroutine 76 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8e29e48, 0xc0013e8690}, 0x82faaf2?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8e66d20, {0x8e29e48, 0xc0013e8690}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x8e29eb8?, 0xc0014043f0?}, {{0x8e42958?, 0xc000d9c600?}, 0xc000798b30?}, 0xc00008f760, {0xc000080990, 0x13}, {0x8319ec1, 0x12}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:852\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0xc0013f6c97, 0x7}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:346\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func9.2({0x7fdc50f03d38, 0xc000baa1e0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:74\n                | \n                | By(\"creating the HCP cluster\")\n                \u003e err = tc.CreateHCPClusterFromParam(ctx,\n                | \tGinkgoLogr,\n                | \t*resourceGroup.Name,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8e42a30?, 0xc000baa1e0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 96\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80 @ 05/14/26 05:30:00.526\n  \"ts\"=\"2026-05-14 05:30:00.526801\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-05-14 05:30:00.526855\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-hcp-cluster-np-128-xgjwjl\" \"resourceGroup\"=\"clusternp128-dw5kpk\"\n  \"ts\"=\"2026-05-14 05:30:00.645848\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-hcp-cluster-np-128-xgjwjl\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-05-14 05:30:00.988854\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-a85d76e69c62fcc4973899bcc5957cf5e02900c7383d8dbab637c1fde0fca561.yaml.gz\"\n  \"ts\"=\"2026-05-14 05:30:00.988958\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-a85d76e69c62fcc4973899bcc5957cf5e02900c7383d8dbab637c1fde0fca561.yaml.gz\"\n  \"ts\"=\"2026-05-14 05:30:00.989012\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-05-14 05:30:00.991791\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-16\"\n  \"ts\"=\"2026-05-14 05:30:00.991804\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"clusternp128-dw5kpk\"\n  \"ts\"=\"2026-05-14 05:30:01.307927\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"clusternp128-dw5kpk\" \"deployment\"=\"pooledNonMsiScopedAssignments-3ljyjjx2n7gxo\"\n  \"ts\"=\"2026-05-14 05:30:01.324197\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-16\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-05-14 05:30:01.368575\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"clusternp128-dw5kpk\" \"deployment\"=\"customer-infra-hcp-cluster-np-128-xgjwjl\"\n  \"ts\"=\"2026-05-14 05:30:01.445985\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-16\" \"deployment\"=\"msi-deployment-16\"\n  \"ts\"=\"2026-05-14 05:30:01.570429\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-05-14 05:30:01.771943\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-05-14 05:30:01.772204\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"clusternp128-dw5kpk\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","remaining specs.  Interrupt again to skip cleanup.\n    Here's a current progress report:\n      Customer should be able to create node pools with ARM64-based VMs (Spec Runtime: 2h29m59.866s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:35\n        In [It] (Node Runtime: 2h29m59.866s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:35\n          At [By Step] creating the HCP cluster (Step Runtime: 13m47.243s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:82\n\n          Spec Goroutine\n          goroutine 68 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8e29e48, 0xc0002211d0}, 0x82faaf2?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8e66d20, {0x8e29e48, 0xc0002211d0}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x8e29eb8?, 0xc00059c000?}, {{0x8e42958?, 0xc000e5de00?}, 0xc000e2f1d0?}, 0xc00008f768, {0xc0015403a8, 0x17}, {0x8325b3f, 0x14}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:852\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0xc0012fb230, 0x7}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:346\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func4.2({0x7f50c840d880, 0xc000d3a480})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:83\n                | \n                | By(\"creating the HCP cluster\")\n                \u003e err = tc.CreateHCPClusterFromParam(\n                | \tctx,\n                | \tGinkgoLogr,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8e42a30?, 0xc000d3a480?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 100\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90 @ 05/14/26 05:30:00.521\n  \"ts\"=\"2026-05-14 05:30:00.521490\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-05-14 05:30:00.521588\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-arm64-vm-hcp-cluster-g28xxb\" \"resourceGroup\"=\"arm64-vm-cluster-5c9nb2\"\n  \"ts\"=\"2026-05-14 05:30:00.632760\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-arm64-vm-hcp-cluster-g28xxb\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-05-14 05:30:01.028526\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-a6eb9d68aa9cc22f7e78141cf691002056d2ca6a7d34a8fba7d067b26fef9e86.yaml.gz\"\n  \"ts\"=\"2026-05-14 05:30:01.028613\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-a6eb9d68aa9cc22f7e78141cf691002056d2ca6a7d34a8fba7d067b26fef9e86.yaml.gz\"\n  \"ts\"=\"2026-05-14 05:30:01.028662\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-05-14 05:30:01.030090\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-61\"\n  \"ts\"=\"2026-05-14 05:30:01.030091\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"arm64-vm-cluster-5c9nb2\"\n  \"ts\"=\"2026-05-14 05:30:01.333150\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-61\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-05-14 05:30:01.402843\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"arm64-vm-cluster-5c9nb2\" \"deployment\"=\"pooledNonMsiScopedAssignments-6htyoibevj7ye\"\n  \"ts\"=\"2026-05-14 05:30:01.445386\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-61\" \"deployment\"=\"msi-deployment-61\"\n  \"ts\"=\"2026-05-14 05:30:01.468401\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"arm64-vm-cluster-5c9nb2\" \"deployment\"=\"customer-infra-arm64-vm-hcp-cluster-g28xxb\"\n  \"ts\"=\"2026-05-14 05:30:01.574116\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-05-14 05:30:01.760665\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-05-14 05:30:01.763535\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"arm64-vm-cluster-5c9nb2\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nremaining specs.  Interrupt again to skip cleanup.\n    Here's a current progress report:\n      Customer should be able to create node pools with ARM64-based VMs (Spec Runtime: 2h29m59.866s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:35\n        In [It] (Node Runtime: 2h29m59.866s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:35\n          At [By Step] creating the HCP cluster (Step Runtime: 13m47.243s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:82\n\n          Spec Goroutine\n          goroutine 68 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8e29e48, 0xc0002211d0}, 0x82faaf2?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8e66d20, {0x8e29e48, 0xc0002211d0}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x8e29eb8?, 0xc00059c000?}, {{0x8e42958?, 0xc000e5de00?}, 0xc000e2f1d0?}, 0xc00008f768, {0xc0015403a8, 0x17}, {0x8325b3f, 0x14}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:852\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0xc0012fb230, 0x7}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:346\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func4.2({0x7f50c840d880, 0xc000d3a480})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:83\n                | \n                | By(\"creating the HCP cluster\")\n                \u003e err = tc.CreateHCPClusterFromParam(\n                | \tctx,\n                | \tGinkgoLogr,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8e42a30?, 0xc000d3a480?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 100\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90 @ 05/14/26 05:30:00.521\n  \"ts\"=\"2026-05-14 05:30:00.521490\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-05-14 05:30:00.521588\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-arm64-vm-hcp-cluster-g28xxb\" \"resourceGroup\"=\"arm64-vm-cluster-5c9nb2\"\n  \"ts\"=\"2026-05-14 05:30:00.632760\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-arm64-vm-hcp-cluster-g28xxb\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-05-14 05:30:01.028526\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-a6eb9d68aa9cc22f7e78141cf691002056d2ca6a7d34a8fba7d067b26fef9e86.yaml.gz\"\n  \"ts\"=\"2026-05-14 05:30:01.028613\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-a6eb9d68aa9cc22f7e78141cf691002056d2ca6a7d34a8fba7d067b26fef9e86.yaml.gz\"\n  \"ts\"=\"2026-05-14 05:30:01.028662\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-05-14 05:30:01.030090\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-61\"\n  \"ts\"=\"2026-05-14 05:30:01.030091\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"arm64-vm-cluster-5c9nb2\"\n  \"ts\"=\"2026-05-14 05:30:01.333150\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-61\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-05-14 05:30:01.402843\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"arm64-vm-cluster-5c9nb2\" \"deployment\"=\"pooledNonMsiScopedAssignments-6htyoibevj7ye\"\n  \"ts\"=\"2026-05-14 05:30:01.445386\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-61\" \"deployment\"=\"msi-deployment-61\"\n  \"ts\"=\"2026-05-14 05:30:01.468401\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"arm64-vm-cluster-5c9nb2\" \"deployment\"=\"customer-infra-arm64-vm-hcp-cluster-g28xxb\"\n  \"ts\"=\"2026-05-14 05:30:01.574116\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-05-14 05:30:01.760665\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-05-14 05:30:01.763535\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"arm64-vm-cluster-5c9nb2\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","nt progress report:\n      Customer should be able to create an HCP cluster with custom autoscaling (Spec Runtime: 2h29m59.884s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:36\n        In [It] (Node Runtime: 2h29m59.884s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:36\n          At [By Step] creating the cluster (Step Runtime: 13m58.964s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:90\n\n          Spec Goroutine\n          goroutine 114 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8e29e48, 0xc000bfafa0}, 0x82faaf2?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8e66d20, {0x8e29e48, 0xc000bfafa0}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x8e29eb8?, 0xc000946150?}, {{0x8e42958?, 0xc000933000?}, 0xc0014c2e28?}, 0xc00008f720, {0xc00153b140, 0x1a}, {0x833717c, 0x17}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:852\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0xc0013d3c47, 0x7}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:346\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func6.2({0x7f6588241860, 0xc000e02f00})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:91\n                | \n                | By(\"creating the cluster\")\n                \u003e err = tc.CreateHCPClusterFromParam(ctx,\n                | \tGinkgoLogr,\n                | \t*resourceGroup.Name,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8e42a30?, 0xc000e02f00?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 111\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97 @ 05/14/26 05:30:00.521\n  \"ts\"=\"2026-05-14 05:30:00.521474\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-05-14 05:30:00.521545\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-autoscaling-hcp-cluster-g9ldv2\" \"resourceGroup\"=\"autoscaling-cluster-lfddg6\"\n  \"ts\"=\"2026-05-14 05:30:00.618320\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-autoscaling-hcp-cluster-g9ldv2\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-05-14 05:30:02.019461\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-31a71a35884d87a4a1de36eceeb324011e1c775154472da33ffc40cde64c08cb.yaml.gz\"\n  \"ts\"=\"2026-05-14 05:30:02.019562\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-31a71a35884d87a4a1de36eceeb324011e1c775154472da33ffc40cde64c08cb.yaml.gz\"\n  \"ts\"=\"2026-05-14 05:30:02.019614\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-05-14 05:30:02.021044\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"autoscaling-cluster-lfddg6\"\n  \"ts\"=\"2026-05-14 05:30:02.021025\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-12\"\n  \"ts\"=\"2026-05-14 05:30:02.340783\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"autoscaling-cluster-lfddg6\" \"deployment\"=\"pooledNonMsiScopedAssignments-e4ixwyxsfjnje\"\n  \"ts\"=\"2026-05-14 05:30:02.389983\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"autoscaling-cluster-lfddg6\" \"deployment\"=\"customer-infra-autoscaling-hcp-cluster-g9ldv2\"\n  \"ts\"=\"2026-05-14 05:30:02.843289\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-12\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-05-14 05:30:03.312787\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-12\" \"deployment\"=\"msi-deployment-12\"\n  \"ts\"=\"2026-05-14 05:30:03.420865\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-05-14 05:30:04.407214\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-05-14 05:30:04.407388\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"autoscaling-cluster-lfddg6\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nnt progress report:\n      Customer should be able to create an HCP cluster with custom autoscaling (Spec Runtime: 2h29m59.884s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:36\n        In [It] (Node Runtime: 2h29m59.884s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:36\n          At [By Step] creating the cluster (Step Runtime: 13m58.964s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:90\n\n          Spec Goroutine\n          goroutine 114 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8e29e48, 0xc000bfafa0}, 0x82faaf2?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8e66d20, {0x8e29e48, 0xc000bfafa0}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x8e29eb8?, 0xc000946150?}, {{0x8e42958?, 0xc000933000?}, 0xc0014c2e28?}, 0xc00008f720, {0xc00153b140, 0x1a}, {0x833717c, 0x17}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:852\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0xc0013d3c47, 0x7}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:346\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func6.2({0x7f6588241860, 0xc000e02f00})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:91\n                | \n                | By(\"creating the cluster\")\n                \u003e err = tc.CreateHCPClusterFromParam(ctx,\n                | \tGinkgoLogr,\n                | \t*resourceGroup.Name,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8e42a30?, 0xc000e02f00?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 111\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97 @ 05/14/26 05:30:00.521\n  \"ts\"=\"2026-05-14 05:30:00.521474\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-05-14 05:30:00.521545\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-autoscaling-hcp-cluster-g9ldv2\" \"resourceGroup\"=\"autoscaling-cluster-lfddg6\"\n  \"ts\"=\"2026-05-14 05:30:00.618320\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-autoscaling-hcp-cluster-g9ldv2\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-05-14 05:30:02.019461\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-31a71a35884d87a4a1de36eceeb324011e1c775154472da33ffc40cde64c08cb.yaml.gz\"\n  \"ts\"=\"2026-05-14 05:30:02.019562\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-31a71a35884d87a4a1de36eceeb324011e1c775154472da33ffc40cde64c08cb.yaml.gz\"\n  \"ts\"=\"2026-05-14 05:30:02.019614\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-05-14 05:30:02.021044\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"autoscaling-cluster-lfddg6\"\n  \"ts\"=\"2026-05-14 05:30:02.021025\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-12\"\n  \"ts\"=\"2026-05-14 05:30:02.340783\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"autoscaling-cluster-lfddg6\" \"deployment\"=\"pooledNonMsiScopedAssignments-e4ixwyxsfjnje\"\n  \"ts\"=\"2026-05-14 05:30:02.389983\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"autoscaling-cluster-lfddg6\" \"deployment\"=\"customer-infra-autoscaling-hcp-cluster-g9ldv2\"\n  \"ts\"=\"2026-05-14 05:30:02.843289\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-12\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-05-14 05:30:03.312787\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-12\" \"deployment\"=\"msi-deployment-12\"\n  \"ts\"=\"2026-05-14 05:30:03.420865\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-05-14 05:30:04.407214\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-05-14 05:30:04.407388\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"autoscaling-cluster-lfddg6\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"3ee7737356a84a0c5e11c2d4e419cda36fea174ff09da91a87fc064ae092679c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"a4048dddf6e2231629b217ebb58ea4d448c77d328c5082e5daa6e31b4130c7ec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"b0f1dd579f12ef5a127a84a1cc98337fd6196f180d8bab9000a7cadfd2bb7170","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e9c6c185c09aa6c23477c53c1d3d7fb1bb60e041c3cb718558ce145b6718d4d","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":6.25,"anchor_occurrences":2,"after_last_push_count":11,"trend_counts":[0,0,0,0,1,1,0,0],"trend_range":"2026-05-10..2026-05-17 UTC","prior_windows_present":3,"prior_window_starts":["2026-04-20","2026-04-27","2026-05-04"],"prior_runs_affected":11,"prior_last_seen_at":"2026-05-07T21:40:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 2700.007s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 2700.007s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"0c4ed9bba8fe112ea7971d2241bf4fe4d2d0df6edae20700d459cff3d63baf26","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054754184330219520","occurred_at":"2026-05-14T02:42:27Z","signature_id":"f816448949a54c698daec440b6be14869199dd5156942a339f2ff5a445aaf853","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"0fa92f751490b05776d8c9d0ffad7e078841d629f2bbb5cf7b86f379f22f1b39","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message timed out after 29s waiting for resolution of desired version from '4.21.15' cluster version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":6.25,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,2,0,0,0,0],"trend_range":"2026-05-10..2026-05-17 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:160]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0014084a0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-5wk752\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-qwr4c4\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/423d54f5-9311-40ba-a0fa-52e5d5e4dcd8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/423d54f5-9311-40ba-a0fa-52e5d5e4dcd8\",\n      \"name\": \"423d54f5-9311-40ba-a0fa-52e5d5e4dcd8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-05-13T13:46:24.415720989Z\",\n      \"endTime\": \"2026-05-13T13:48:12.932582258Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"timed out after 29s waiting for resolution of desired version from '4.21.15' cluster version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:160]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0014084a0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-5wk752\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-qwr4c4\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/423d54f5-9311-40ba-a0fa-52e5d5e4dcd8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/423d54f5-9311-40ba-a0fa-52e5d5e4dcd8\",\n      \"name\": \"423d54f5-9311-40ba-a0fa-52e5d5e4dcd8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-05-13T13:46:24.415720989Z\",\n      \"endTime\": \"2026-05-13T13:48:12.932582258Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"timed out after 29s waiting for resolution of desired version from '4.21.15' cluster version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:160]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000598220\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-fwht97\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-wn6jfz\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4f5e233c-1544-4fcf-acfe-ac988c79aefa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4f5e233c-1544-4fcf-acfe-ac988c79aefa\",\n      \"name\": \"4f5e233c-1544-4fcf-acfe-ac988c79aefa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-05-13T02:53:03.508575142Z\",\n      \"endTime\": \"2026-05-13T02:54:22.582271334Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"timed out after 29s waiting for resolution of desired version from '4.21.15' cluster version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:160]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000598220\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-fwht97\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-wn6jfz\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4f5e233c-1544-4fcf-acfe-ac988c79aefa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4f5e233c-1544-4fcf-acfe-ac988c79aefa\",\n      \"name\": \"4f5e233c-1544-4fcf-acfe-ac988c79aefa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-05-13T02:53:03.508575142Z\",\n      \"endTime\": \"2026-05-13T02:54:22.582271334Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"timed out after 29s waiting for resolution of desired version from '4.21.15' cluster version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"32c5eaec9aee930a7246240a8b34ba30a49776a7d954a5f9bec99120da53877c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2054382242708328448","occurred_at":"2026-05-13T02:00:06Z","signature_id":"32450619c037bc494079db40c33ca0044f5df5558d9030d23cf045bacda24022","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-wm4xp2, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wm4xp2\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.125,"anchor_occurrences":1,"after_last_push_count":8,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-05-10..2026-05-17 UTC","prior_windows_present":4,"prior_window_starts":["2026-04-13","2026-04-20","2026-04-27","2026-05-04"],"prior_runs_affected":10,"prior_last_seen_at":"2026-05-09T09:16:57Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:111]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001891b30\u003e: \n    failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-wm4xp2, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wm4xp2\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:111]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001891b30\u003e: \n    failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-wm4xp2, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wm4xp2\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055186423362883584","occurred_at":"2026-05-15T07:20:01Z","signature_id":"fe4e75e492c736e53377399337ec4686a6bb5393911c09b43f362a0ad556dfce","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"ae6a7f7d2e3c3444ed20150a67378f751586e201de3a927f677f72a871b2bf44","failure_pattern":"DeleteResourceGroup timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-zg8gxb, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.125,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-05-10..2026-05-17 UTC","prior_windows_present":1,"prior_window_starts":["2026-04-20"],"prior_runs_affected":2,"prior_last_seen_at":"2026-04-24T18:17:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:143]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000490e40\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-zg8gxb, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:143]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000490e40\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-zg8gxb, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"b8bd9012025eb327d93ee57bab10dac7f7e9844f919d1392e7bf6f9a26973f0d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d6c1dec4d8c20fbbd7f2b826c14447c68052c707edbc8727865c6a1f97f0a0dd","failure_pattern":"failed creating deployment \"test-vm\" in resourcegroup=\"\u003cresource-group\u003e\": PUT \u003curl\u003e","search_query":"failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-75qph4\": PUT https://management","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.125,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-05-10..2026-05-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:114]: Timed out after 1800.007s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000c06c60\u003e: \n    failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-75qph4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-75qph4/providers/Microsoft.Resources/deployments/test-vm\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidTemplateDeployment\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidTemplateDeployment\",\n        \"message\": \"The template deployment 'test-vm' is not valid according to the validation procedure. The tracking id is 'f01f19bc-725b-43ef-8fbf-4230b634f3a7'. See inner errors for details.\",\n        \"details\": [\n          {\n            \"code\": \"ResourceCountExceedsLimitDueToTemplate\",\n            \"message\": \"Subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has a quota of 300 for resources of type PublicIpAddress with sku SkuNotSpecified. Subscription currently has 300 resources and the template contains 1 new resources of the this type which exceeds the quota. Please contact support to increase the quota for resource type PublicIpAddress\",\n            \"details\": []\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:114]: Timed out after 1800.007s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000c06c60\u003e: \n    failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-75qph4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-75qph4/providers/Microsoft.Resources/deployments/test-vm\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidTemplateDeployment\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidTemplateDeployment\",\n        \"message\": \"The template deployment 'test-vm' is not valid according to the validation procedure. The tracking id is 'f01f19bc-725b-43ef-8fbf-4230b634f3a7'. See inner errors for details.\",\n        \"details\": [\n          {\n            \"code\": \"ResourceCountExceedsLimitDueToTemplate\",\n            \"message\": \"Subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has a quota of 300 for resources of type PublicIpAddress with sku SkuNotSpecified. Subscription currently has 300 resources and the template contains 1 new resources of the this type which exceeds the quota. Please contact support to increase the quota for resource type PublicIpAddress\",\n            \"details\": []\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055188570250940416","occurred_at":"2026-05-15T07:28:33Z","signature_id":"de0afbdb6a0cbda24c652c13013470e5e4d8053a540deb9d09b9e67809368f0b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"7137d3f115bf4c264bb74c888bdc6c3954a56a37032931fb9aa8d47ff31a4953","failure_pattern":"x509: certificate signed by unknown authority","search_query":"x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.125,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-05-10..2026-05-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:155]: Timed out after 600.002s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:155]: Timed out after 600.002s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2055454507260710912","occurred_at":"2026-05-16T01:05:17Z","signature_id":"85944d4fa3cc8f9a02514637d6119a2fd69da50caa8d14fdae25e1e32d2d5f52","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}