{"meta":{"start_date":"2026-03-25","end_date":"2026-03-25","start_at":"2026-03-25T00:00:00Z","end_at":"2026-03-26T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T21:41:10Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":6,"failed_runs":3,"raw_occurrences":51,"matched_occurrences":51,"runs_affected":3},"rows":[{"environment":"int","failure_pattern_id":"0ece011306dab74cd22dd19f1f3c89bbdb930a2321b61be59e4a6a90aae6e3eb","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-f49ng4, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-f49ng4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-f49ng4, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":33,"runs_affected":2,"failed_runs":2,"run_impact_percent":33.333333333333336,"anchor_occurrences":33,"after_last_push_count":33,"trend_counts":[0,0,0,0,0,0,33],"trend_range":"2026-03-19..2026-03-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-02-23","2026-03-02","2026-03-09","2026-03-16"],"prior_runs_affected":20,"prior_last_seen_at":"2026-03-22T04:10:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes vm type NC4asT4v3 in a single cluster","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000dfc6c0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-k4pjbd\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-k4pjbd, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-k4pjbd\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-k4pjbd, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001132750\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-k4pjbd\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000dfc6c0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-k4pjbd\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-k4pjbd, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-k4pjbd\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-k4pjbd, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001132750\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-k4pjbd\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f845a0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-qf7nf4, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-qf7nf4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-qf7nf4, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-qf7nf4, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-qf7nf4\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-qf7nf4, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0006b9680\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-qf7nf4\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000f84570\u003e{\n                msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-qf7nf4\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-qf7nf4, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0006b9680\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-qf7nf4\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f845a0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-qf7nf4, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-qf7nf4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-qf7nf4, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-qf7nf4, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-qf7nf4\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-qf7nf4, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0006b9680\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-qf7nf4\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000f84570\u003e{\n                msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-qf7nf4\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-qf7nf4, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0006b9680\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-qf7nf4\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009aa0c0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-9l9jxt, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-9l9jxt\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-9l9jxt, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-9l9jxt, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-9l9jxt\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-9l9jxt, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000ffcdc0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-9l9jxt\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0009aa090\u003e{\n                msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-9l9jxt\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-9l9jxt, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000ffcdc0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-9l9jxt\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009aa0c0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-9l9jxt, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-9l9jxt\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-9l9jxt, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-9l9jxt, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-9l9jxt\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-9l9jxt, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000ffcdc0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-9l9jxt\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0009aa090\u003e{\n                msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-9l9jxt\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-9l9jxt, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000ffcdc0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-9l9jxt\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036937715248795648","occurred_at":"2026-03-25T22:46:10Z","signature_id":"0abd6ba6a343b25ac6bc2b9090d6dd4c43600ae4596b878201b68717d46c10f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036937715248795648","occurred_at":"2026-03-25T22:46:10Z","signature_id":"2fa69ac033ec2c159e3bd9b2e40e3daffa0345185abca8e7eedea82465ba75da","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036937715248795648","occurred_at":"2026-03-25T22:46:10Z","signature_id":"396b46c32351385a47482299257f05b37ffa9eb30edd82ede40f2182751c744c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036937715248795648","occurred_at":"2026-03-25T22:46:10Z","signature_id":"3e92754240236aa292142c1d6a30b07a5d2d04723d32f85f3c0cf2b5794c47a2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036937715248795648","occurred_at":"2026-03-25T22:46:10Z","signature_id":"5e1b072c5c4efd46b9f62c161e0a7089c814e2137b3962e693229e465706f72f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036937715248795648","occurred_at":"2026-03-25T22:46:10Z","signature_id":"72aa56dff939c6025f4f7c11e3c5049a1a20af51e8f30ca9860456dbfb9af6e2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036937715248795648","occurred_at":"2026-03-25T22:46:10Z","signature_id":"890750bded37d69b41e0f34fb6df46f3fc1054b5140869199ec32a5b3081eb4c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036937715248795648","occurred_at":"2026-03-25T22:46:10Z","signature_id":"8e56855e1db8724cc63db64aa3b756c866d72cf3451486f271e16ef5f999e885","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036937715248795648","occurred_at":"2026-03-25T22:46:10Z","signature_id":"99fff06e06fa4e746dd12d3295fe5db02b356c23490d75b74177af4d2dea5916","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036937715248795648","occurred_at":"2026-03-25T22:46:10Z","signature_id":"9ca7282a6c9148a16f747bb89a2792d09f255dbd90095b376be5b11d7369c21c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036937715248795648","occurred_at":"2026-03-25T22:46:10Z","signature_id":"9e2f8d8ccf16e8471ed5d68593deb6c7a651ca43f32e3611fc5fc5847d5c9643","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036937715248795648","occurred_at":"2026-03-25T22:46:10Z","signature_id":"b65c32fc3076ed21c7e9e463999dc4d15213f50d2fa6d44145eb34632546066e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036937715248795648","occurred_at":"2026-03-25T22:46:10Z","signature_id":"b69ff14a6bbec0fc0332f1b260135c84319cc4d6f0f555fe2ea799d4bd61b18e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036937715248795648","occurred_at":"2026-03-25T22:46:10Z","signature_id":"c072d5a9f22f01b7f36adab419810d5c0611cf22a79da54783fcc25edf6fd8b0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036937715248795648","occurred_at":"2026-03-25T22:46:10Z","signature_id":"e65ec5171a81f8cf7f2302987d1b0338cb7427e4cb9166f4b23651a1bc8eb9b2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036937715248795648","occurred_at":"2026-03-25T22:46:10Z","signature_id":"e76543169564f848d2e0801d432ef455ad281ac6b740c2df8e8032ac19d82de0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036937715248795648","occurred_at":"2026-03-25T22:46:10Z","signature_id":"efed81635e50d9d1c4068cfb6e64585dd71bba11b16834d3dfc467faffa45eb0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036888953293901824","occurred_at":"2026-03-25T19:32:20Z","signature_id":"620ca66870a34e97b6750125e9b9e14b7b8fe01f59a785f8bfd0d4cedcd5c68b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036888953293901824","occurred_at":"2026-03-25T19:32:20Z","signature_id":"6a5653100a15735283a095cf13a7321b6982628823049ac8bbfe7720f48a4eb7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036888953293901824","occurred_at":"2026-03-25T19:32:20Z","signature_id":"6c824ea9cf099b09e42d6513f7c92aaa97fc5ec06ec5ba5a3d17042e7eb468f2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036888953293901824","occurred_at":"2026-03-25T19:32:20Z","signature_id":"720d13e12efdeb1e777a7a024dd81578ae984b2e2477b40f852098e013812518","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036888953293901824","occurred_at":"2026-03-25T19:32:20Z","signature_id":"7f8901b59629097cb77ba70560c73cc19224a67ec6ed6792cb4fca9f71a3bb05","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036888953293901824","occurred_at":"2026-03-25T19:32:20Z","signature_id":"8e4f90db70e32349f5d8a9bb793d483ccb12a00a27e42ec4cf55ac0ad81f90b3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036888953293901824","occurred_at":"2026-03-25T19:32:20Z","signature_id":"956eb8f30814e3e07f5a999e7f9621d8a77fe5cdd20db749081ebe4c010d7d9d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036888953293901824","occurred_at":"2026-03-25T19:32:20Z","signature_id":"a4139f0082ca30247e31673c5b017d7ecc0cd9e6c91a06fea1f652a33429cce4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036888953293901824","occurred_at":"2026-03-25T19:32:20Z","signature_id":"a7e82d90e306821a63ffd4e24dc8f108cc1342c9197e75ce53bbebc9d9f740a7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036888953293901824","occurred_at":"2026-03-25T19:32:20Z","signature_id":"a9a8df33d17e4ea9ac92e2ec93cbf1001a32138adfc5e86e5ff87984e41d2206","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036888953293901824","occurred_at":"2026-03-25T19:32:20Z","signature_id":"b64eaf8a601f7825b32a1a3481a45634ab63cd57a1eeea857a82e63f491a85aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036888953293901824","occurred_at":"2026-03-25T19:32:20Z","signature_id":"c23865cd5849ff60831243bed0a39800cfe2abe39bf4d37bf13c249d3191e91d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036888953293901824","occurred_at":"2026-03-25T19:32:20Z","signature_id":"d4a258a5dda6a50ec1fcba269c392b1e4116951d9db4f567ec17e0bf2b493b44","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036888953293901824","occurred_at":"2026-03-25T19:32:20Z","signature_id":"e6f6f8747a2a6b20b7647fa188c0951d43966d43f803abe20031fa5d7bcc502e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036888953293901824","occurred_at":"2026-03-25T19:32:20Z","signature_id":"ebda800b88a1221e8974af32e35b9d4bc895d36522b2c36e2a7f528136906f0e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036888953293901824","occurred_at":"2026-03-25T19:32:20Z","signature_id":"f20a06cc299b45dd74ccda10acf7995d57bfb569e11f58f9f6d169c8126b694f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"ed91849277e23f84f2cfe3e0bb2a2379781cf9007095371a6220d51d0605b4b7","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","occurrences":12,"runs_affected":2,"failed_runs":2,"run_impact_percent":33.333333333333336,"anchor_occurrences":12,"after_last_push_count":33,"trend_counts":[0,0,0,0,0,4,12],"trend_range":"2026-03-19..2026-03-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-02-23","2026-03-02","2026-03-09"],"prior_runs_affected":16,"prior_last_seen_at":"2026-03-11T00:05:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:127]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:127]: Interrupted by User","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:122]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:122]: Interrupted by User","fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:37]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:37]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036937715248795648","occurred_at":"2026-03-25T22:46:10Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036937715248795648","occurred_at":"2026-03-25T22:46:10Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036937715248795648","occurred_at":"2026-03-25T22:46:10Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036937715248795648","occurred_at":"2026-03-25T22:46:10Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036937715248795648","occurred_at":"2026-03-25T22:46:10Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036937715248795648","occurred_at":"2026-03-25T22:46:10Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036888953293901824","occurred_at":"2026-03-25T19:32:20Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036888953293901824","occurred_at":"2026-03-25T19:32:20Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036888953293901824","occurred_at":"2026-03-25T19:32:20Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036888953293901824","occurred_at":"2026-03-25T19:32:20Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036888953293901824","occurred_at":"2026-03-25T19:32:20Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036888953293901824","occurred_at":"2026-03-25T19:32:20Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"2d1906ec85d934b5bfe9201fdd489d4450037751378f0d66a8abd6c1b1f5a4bd","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":33.333333333333336,"anchor_occurrences":2,"after_last_push_count":7,"trend_counts":[0,0,0,0,0,1,2],"trend_range":"2026-03-19..2026-03-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-02-23","2026-03-02"],"prior_runs_affected":13,"prior_last_seen_at":"2026-03-08T23:23:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.000s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.000s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: false\nto be true","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: false\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036937715248795648","occurred_at":"2026-03-25T22:46:10Z","signature_id":"7dc26f820fd2928c111033bd0ce08af1bae6042a6cbe5fb5bf76d461b5be1d01","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036888953293901824","occurred_at":"2026-03-25T19:32:20Z","signature_id":"b8285d1fe5e0e06b0118077c74385230269c41bde5a189b9513acacc0d1a338e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"d0993ee7c174e66d21d6873cdcbb83cae1352c1482345816cea0f68947670c76","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":33.333333333333336,"anchor_occurrences":2,"after_last_push_count":20,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-03-19..2026-03-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-02","2026-03-16"],"prior_runs_affected":7,"prior_last_seen_at":"2026-03-18T07:37:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["4993c0?, 0x0?, 0x0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:349 +0x905 fp=0xc000bbc960 sp=0xc000bbbd58 pc=0x2fabdc5\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runSpecs(0xc00082ca88, {0x0, 0x0}, {0xb57c4c0, 0x0, 0x0}, {0xc00007a094, 0x2a}, 0x0, {0xc000499380, ...})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:496 +0xa9b fp=0xc000bbce10 sp=0xc000bbc960 pc=0x2fbf8bb\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).RunSpec(_, {_, _}, {_, _, _}, {_, _}, {0xc00007a094, 0x2a}, ...)\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite_patch.go:68 +0x3c5 fp=0xc000bbceb8 sp=0xc000bbce10 pc=0x2fc6c25\ngithub.com/openshift-eng/openshift-tests-extension/pkg/ginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite.func1.1({0x2ae0a85?, 0xc0000eade0?})\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/ginkgo/util.go:85 +0x2ba fp=0xc000bbdb88 sp=0xc000bbceb8 pc=0x538025a\ngithub.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.runSpec({0x83c6ff8, 0xc000b3f680}, 0xc000ad1080, 0x1)\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:596 +0xee fp=0xc000bbdc30 sp=0xc000bbdb88 pc=0x53765ce\ngithub.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:237 +0x196 fp=0xc000bbdfe0 sp=0xc000bbdc30 pc=0x5372356\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000bbdfe8 sp=0xc000bbdfe0 pc=0x2b59c01\ncreated by github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:230 +0x1e5\n\ngoroutine 120 gp=0xc000103500 m=nil [select]:\nruntime.gopark(0xc000094e10?, 0x2?, 0xe8?, 0x92?, 0xc000094e04?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000094c90 sp=0xc000094c70 pc=0x2b5148e\nruntime.selectgo(0xc000094e10, 0xc000094e00, 0xc00171a7c0?, 0x0, 0x33500dc?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000094dd0 sp=0xc000094c90 pc=0x2b2c277\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x83c6ff8, 0xc000ea9270}, 0x798f3e3?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36 +0x76 fp=0xc000094e40 sp=0xc000094dd0 pc=0x334f836\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x83fac80, {0x83c6ff8, 0xc000ea9270}, 0x0)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284 +0x7b2 fp=0xc000095188 sp=0xc000094e40 pc=0x3835752\ngithub.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x83c7068?, 0xc001733ea0?}, {{0x83dce08?, 0xc00024e080?}, 0xc00170c9f0?}, 0xc000095680, {0xc000081278, 0x17}, {0x7997e19, 0xe}, ...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:694 +0x405 fp=0xc000095400 sp=0xc000095188 pc=0x4fb57e5\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x796f8c7, 0x4}, ...}, ...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:346 +0x497 fp=0xc000095750 sp=0xc000095400 pc=0x4fa9597\ngithub.com/Azure/ARO-HCP/test/e2e.init.func26.2({0x7fc7a4727a60, 0xc00079f230})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:106 +0x3a6 fp=0xc000095f58 sp=0xc000095750 pc=0x5173386\ngithub.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x83dcee0?, 0xc00079f230?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465 +0x3e fp=0xc000095f88 sp=0xc000095f58 pc=0x2fb163e\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc000095fe0 sp=0xc000095f88 pc=0x2fc5d5b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000095fe8 sp=0xc000095fe0 pc=0x2b59c01\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 117\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nrax    0xca\nrbx    0x0\nrcx    0x2b5ba23\nrdx    0x0\nrdi    0xb551770\nrsi    0x80\nrbp    0x7ffcf5c8ce70\nrsp    0x7ffcf5c8ce28\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x0\nr13    0x1\nr14    0xb54f680\nr15    0x5\nrip    0x2b5ba21\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command\n4993c0?, 0x0?, 0x0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:349 +0x905 fp=0xc000bbc960 sp=0xc000bbbd58 pc=0x2fabdc5\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runSpecs(0xc00082ca88, {0x0, 0x0}, {0xb57c4c0, 0x0, 0x0}, {0xc00007a094, 0x2a}, 0x0, {0xc000499380, ...})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:496 +0xa9b fp=0xc000bbce10 sp=0xc000bbc960 pc=0x2fbf8bb\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).RunSpec(_, {_, _}, {_, _, _}, {_, _}, {0xc00007a094, 0x2a}, ...)\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite_patch.go:68 +0x3c5 fp=0xc000bbceb8 sp=0xc000bbce10 pc=0x2fc6c25\ngithub.com/openshift-eng/openshift-tests-extension/pkg/ginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite.func1.1({0x2ae0a85?, 0xc0000eade0?})\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/ginkgo/util.go:85 +0x2ba fp=0xc000bbdb88 sp=0xc000bbceb8 pc=0x538025a\ngithub.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.runSpec({0x83c6ff8, 0xc000b3f680}, 0xc000ad1080, 0x1)\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:596 +0xee fp=0xc000bbdc30 sp=0xc000bbdb88 pc=0x53765ce\ngithub.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:237 +0x196 fp=0xc000bbdfe0 sp=0xc000bbdc30 pc=0x5372356\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000bbdfe8 sp=0xc000bbdfe0 pc=0x2b59c01\ncreated by github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:230 +0x1e5\n\ngoroutine 120 gp=0xc000103500 m=nil [select]:\nruntime.gopark(0xc000094e10?, 0x2?, 0xe8?, 0x92?, 0xc000094e04?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000094c90 sp=0xc000094c70 pc=0x2b5148e\nruntime.selectgo(0xc000094e10, 0xc000094e00, 0xc00171a7c0?, 0x0, 0x33500dc?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000094dd0 sp=0xc000094c90 pc=0x2b2c277\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x83c6ff8, 0xc000ea9270}, 0x798f3e3?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36 +0x76 fp=0xc000094e40 sp=0xc000094dd0 pc=0x334f836\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x83fac80, {0x83c6ff8, 0xc000ea9270}, 0x0)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284 +0x7b2 fp=0xc000095188 sp=0xc000094e40 pc=0x3835752\ngithub.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x83c7068?, 0xc001733ea0?}, {{0x83dce08?, 0xc00024e080?}, 0xc00170c9f0?}, 0xc000095680, {0xc000081278, 0x17}, {0x7997e19, 0xe}, ...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:694 +0x405 fp=0xc000095400 sp=0xc000095188 pc=0x4fb57e5\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x796f8c7, 0x4}, ...}, ...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:346 +0x497 fp=0xc000095750 sp=0xc000095400 pc=0x4fa9597\ngithub.com/Azure/ARO-HCP/test/e2e.init.func26.2({0x7fc7a4727a60, 0xc00079f230})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:106 +0x3a6 fp=0xc000095f58 sp=0xc000095750 pc=0x5173386\ngithub.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x83dcee0?, 0xc00079f230?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465 +0x3e fp=0xc000095f88 sp=0xc000095f58 pc=0x2fb163e\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc000095fe0 sp=0xc000095f88 pc=0x2fc5d5b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000095fe8 sp=0xc000095fe0 pc=0x2b59c01\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 117\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nrax    0xca\nrbx    0x0\nrcx    0x2b5ba23\nrdx    0x0\nrdi    0xb551770\nrsi    0x80\nrbp    0x7ffcf5c8ce70\nrsp    0x7ffcf5c8ce28\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x0\nr13    0x1\nr14    0xb54f680\nr15    0x5\nrip    0x2b5ba21\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command","a0?, 0x0?, 0x0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:349 +0x905 fp=0xc000e06960 sp=0xc000e05d58 pc=0x2fabdc5\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runSpecs(0xc000d10008, {0x0, 0x0}, {0xb57c4c0, 0x0, 0x0}, {0xc00007a094, 0x2a}, 0x0, {0xc000984780, ...})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:496 +0xa9b fp=0xc000e06e10 sp=0xc000e06960 pc=0x2fbf8bb\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).RunSpec(_, {_, _}, {_, _, _}, {_, _}, {0xc00007a094, 0x2a}, ...)\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite_patch.go:68 +0x3c5 fp=0xc000e06eb8 sp=0xc000e06e10 pc=0x2fc6c25\ngithub.com/openshift-eng/openshift-tests-extension/pkg/ginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite.func1.1({0x2ae0a85?, 0xc000dfea20?})\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/ginkgo/util.go:85 +0x2ba fp=0xc000e07b88 sp=0xc000e06eb8 pc=0x538023a\ngithub.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.runSpec({0x83c6ef8, 0xc000ab5810}, 0xc000b64c00, 0x1)\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:596 +0xee fp=0xc000e07c30 sp=0xc000e07b88 pc=0x53765ae\ngithub.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:237 +0x196 fp=0xc000e07fe0 sp=0xc000e07c30 pc=0x5372336\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000e07fe8 sp=0xc000e07fe0 pc=0x2b59c01\ncreated by github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:230 +0x1e5\n\ngoroutine 92 gp=0xc000181880 m=nil [select]:\nruntime.gopark(0xc000094e50?, 0x2?, 0x20?, 0x26?, 0xc000094e44?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000094cd0 sp=0xc000094cb0 pc=0x2b5148e\nruntime.selectgo(0xc000094e50, 0xc000094e40, 0xc000d8ac00?, 0x0, 0x33500dc?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000094e10 sp=0xc000094cd0 pc=0x2b2c277\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x83c6ef8, 0xc00030e3c0}, 0x798f3e3?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36 +0x76 fp=0xc000094e80 sp=0xc000094e10 pc=0x334f836\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x840bca0, {0x83c6ef8, 0xc00030e3c0}, 0x0)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284 +0x7b2 fp=0xc0000951c8 sp=0xc000094e80 pc=0x37ebd32\ngithub.com/Azure/ARO-HCP/test/util/framework.CreateHCPCluster20251223AndWait({0x83c6f68?, 0xc0004d9810?}, {{0x83dcd08?, 0xc000544b80?}, 0xc00149a558?}, 0xc0017876e0, {0xc000e91760, 0xb}, {0x79b5ee5, 0x14}, ...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:1142 +0x405 fp=0xc000095428 sp=0xc0000951c8 pc=0x4fb9d45\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPCluster20251223FromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x796f8c7, 0x4}, ...}, ...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:389 +0x5d8 fp=0xc0000957b0 sp=0xc000095428 pc=0x4faa0b8\ngithub.com/Azure/ARO-HCP/test/e2e.init.func22.1({0x7f1f2a25a060, 0xc0002af470})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:99 +0x5a5 fp=0xc000095f58 sp=0xc0000957b0 pc=0x516e385\ngithub.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x83dcde0?, 0xc0002af470?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465 +0x3e fp=0xc000095f88 sp=0xc000095f58 pc=0x2fb163e\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc000095fe0 sp=0xc000095f88 pc=0x2fc5d5b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000095fe8 sp=0xc000095fe0 pc=0x2b59c01\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 90\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nrax    0xca\nrbx    0x0\nrcx    0x2b5ba23\nrdx    0x0\nrdi    0xb551770\nrsi    0x80\nrbp    0x7ffcf63bb6d8\nrsp    0x7ffcf63bb690\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x62\nr13    0x1\nr14    0xb54f680\nr15    0x3\nrip    0x2b5ba21\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command\na0?, 0x0?, 0x0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:349 +0x905 fp=0xc000e06960 sp=0xc000e05d58 pc=0x2fabdc5\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runSpecs(0xc000d10008, {0x0, 0x0}, {0xb57c4c0, 0x0, 0x0}, {0xc00007a094, 0x2a}, 0x0, {0xc000984780, ...})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:496 +0xa9b fp=0xc000e06e10 sp=0xc000e06960 pc=0x2fbf8bb\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).RunSpec(_, {_, _}, {_, _, _}, {_, _}, {0xc00007a094, 0x2a}, ...)\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite_patch.go:68 +0x3c5 fp=0xc000e06eb8 sp=0xc000e06e10 pc=0x2fc6c25\ngithub.com/openshift-eng/openshift-tests-extension/pkg/ginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite.func1.1({0x2ae0a85?, 0xc000dfea20?})\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/ginkgo/util.go:85 +0x2ba fp=0xc000e07b88 sp=0xc000e06eb8 pc=0x538023a\ngithub.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.runSpec({0x83c6ef8, 0xc000ab5810}, 0xc000b64c00, 0x1)\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:596 +0xee fp=0xc000e07c30 sp=0xc000e07b88 pc=0x53765ae\ngithub.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:237 +0x196 fp=0xc000e07fe0 sp=0xc000e07c30 pc=0x5372336\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000e07fe8 sp=0xc000e07fe0 pc=0x2b59c01\ncreated by github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:230 +0x1e5\n\ngoroutine 92 gp=0xc000181880 m=nil [select]:\nruntime.gopark(0xc000094e50?, 0x2?, 0x20?, 0x26?, 0xc000094e44?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000094cd0 sp=0xc000094cb0 pc=0x2b5148e\nruntime.selectgo(0xc000094e50, 0xc000094e40, 0xc000d8ac00?, 0x0, 0x33500dc?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000094e10 sp=0xc000094cd0 pc=0x2b2c277\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x83c6ef8, 0xc00030e3c0}, 0x798f3e3?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36 +0x76 fp=0xc000094e80 sp=0xc000094e10 pc=0x334f836\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x840bca0, {0x83c6ef8, 0xc00030e3c0}, 0x0)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284 +0x7b2 fp=0xc0000951c8 sp=0xc000094e80 pc=0x37ebd32\ngithub.com/Azure/ARO-HCP/test/util/framework.CreateHCPCluster20251223AndWait({0x83c6f68?, 0xc0004d9810?}, {{0x83dcd08?, 0xc000544b80?}, 0xc00149a558?}, 0xc0017876e0, {0xc000e91760, 0xb}, {0x79b5ee5, 0x14}, ...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:1142 +0x405 fp=0xc000095428 sp=0xc0000951c8 pc=0x4fb9d45\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPCluster20251223FromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x796f8c7, 0x4}, ...}, ...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:389 +0x5d8 fp=0xc0000957b0 sp=0xc000095428 pc=0x4faa0b8\ngithub.com/Azure/ARO-HCP/test/e2e.init.func22.1({0x7f1f2a25a060, 0xc0002af470})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:99 +0x5a5 fp=0xc000095f58 sp=0xc0000957b0 pc=0x516e385\ngithub.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x83dcde0?, 0xc0002af470?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465 +0x3e fp=0xc000095f88 sp=0xc000095f58 pc=0x2fb163e\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc000095fe0 sp=0xc000095f88 pc=0x2fc5d5b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000095fe8 sp=0xc000095fe0 pc=0x2b59c01\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 90\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nrax    0xca\nrbx    0x0\nrcx    0x2b5ba23\nrdx    0x0\nrdi    0xb551770\nrsi    0x80\nrbp    0x7ffcf63bb6d8\nrsp    0x7ffcf63bb690\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x62\nr13    0x1\nr14    0xb54f680\nr15    0x3\nrip    0x2b5ba21\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036937715248795648","occurred_at":"2026-03-25T22:46:10Z","signature_id":"a610b9bd385b129260c39cf145486707f28ea7906823aef325b4b0a68da5454f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036888953293901824","occurred_at":"2026-03-25T19:32:20Z","signature_id":"611447263b7c349a23626f1aefe3571c1fe8ee7fb11cc2d8aa0d7010f2228952","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"50c48c88fe4eb95e98afa85c61c4077e079f1b9e83d2b9321ba1c497a35a22ac","failure_pattern":"ERROR CODE: Conflict; detail message Resource is already deleting; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to list HCP clusters without node pools at the resource group level","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-19..2026-03-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-09"],"prior_runs_affected":2,"prior_last_seen_at":"2026-03-09T04:24:18Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to list HCP clusters without node pools at the resource group level","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:245]: Unexpected error:\n    \u003c*errors.joinError | 0xc00141cd38\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-r8tknz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-lkzjxd\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Resource is already deleting\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-r8tknz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-lkzjxd\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0008bb5e0\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-r8tknz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-lkzjxd\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Resource is already deleting\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-r8tknz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-lkzjxd\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                err: \u003c*fmt.wrapError | 0xc0008bb5c0\u003e{\n                    msg: \"at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-r8tknz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-lkzjxd\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Resource is already deleting\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-r8tknz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-lkzjxd\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                    err: \u003c*exported.ResponseError | 0xc00178e6c0\u003e{\n                        ErrorCode: \"Conflict\",\n                        StatusCode: 409,\n                        RawResponse: {\n                            Status: \"409 Conflict\",\n                            StatusCode: 409,\n                            Proto: \"HTTP/2.0\",\n                            ProtoMajor: 2,\n                            ProtoMinor: 0,\n                            Header: {\n                                \"Pragma\": [\"no-cache\"],\n                                \"Expires\": [\"-1\"],\n                                \"X-Ms-Request-Id\": [\n                                    \"28ede0cd-2936-4f2f-8253-cb605ecc9170\",\n                                ],\n                                \"X-Ms-Ratelimit-Remaining-Subscription-Deletes\": [\"199\"],\n                                \"X-Ms-Routing-Request-Id\": [\n                                    \"UKSOUTH:20260325T152452Z:5c77f5c4-ef8b-4b65-ad8a-d8b321c39862\",\n                                ],\n                                \"Strict-Transport-Security\": [\n                                    \"max-age=31536000; includeSubDomains\",\n                                ],\n                                \"Content-Length\": [\"295\"],\n                                \"Content-Type\": [\"application/json\"],\n                                \"X-Ms-Correlation-Request-Id\": [\n                                    \"5c77f5c4-ef8b-4b65-ad8a-d8b321c39862\",\n                                ],\n                                \"Cache-Control\": [\"no-cache\"],\n                                \"X-Ms-Error-Code\": [\"Conflict\"],\n                                \"X-Envoy-Upstream-Service-Time\": [\"1026\"],\n                                \"X-Ms-Operation-Identifier\": [\n                                    \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/uksouth/5c5c915c-65f2-46d6-a279-f918cf82b3b5\",\n                                ],\n                                \"X-Ms-Ratelimit-Remaining-Subscription-Global-Del...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:245]: Unexpected error:\n    \u003c*errors.joinError | 0xc00141cd38\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-r8tknz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-lkzjxd\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Resource is already deleting\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-r8tknz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-lkzjxd\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0008bb5e0\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-r8tknz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-lkzjxd\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Resource is already deleting\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-r8tknz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-lkzjxd\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                err: \u003c*fmt.wrapError | 0xc0008bb5c0\u003e{\n                    msg: \"at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-r8tknz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-lkzjxd\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Resource is already deleting\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-r8tknz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-lkzjxd\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                    err: \u003c*exported.ResponseError | 0xc00178e6c0\u003e{\n                        ErrorCode: \"Conflict\",\n                        StatusCode: 409,\n                        RawResponse: {\n                            Status: \"409 Conflict\",\n                            StatusCode: 409,\n                            Proto: \"HTTP/2.0\",\n                            ProtoMajor: 2,\n                            ProtoMinor: 0,\n                            Header: {\n                                \"Pragma\": [\"no-cache\"],\n                                \"Expires\": [\"-1\"],\n                                \"X-Ms-Request-Id\": [\n                                    \"28ede0cd-2936-4f2f-8253-cb605ecc9170\",\n                                ],\n                                \"X-Ms-Ratelimit-Remaining-Subscription-Deletes\": [\"199\"],\n                                \"X-Ms-Routing-Request-Id\": [\n                                    \"UKSOUTH:20260325T152452Z:5c77f5c4-ef8b-4b65-ad8a-d8b321c39862\",\n                                ],\n                                \"Strict-Transport-Security\": [\n                                    \"max-age=31536000; includeSubDomains\",\n                                ],\n                                \"Content-Length\": [\"295\"],\n                                \"Content-Type\": [\"application/json\"],\n                                \"X-Ms-Correlation-Request-Id\": [\n                                    \"5c77f5c4-ef8b-4b65-ad8a-d8b321c39862\",\n                                ],\n                                \"Cache-Control\": [\"no-cache\"],\n                                \"X-Ms-Error-Code\": [\"Conflict\"],\n                                \"X-Envoy-Upstream-Service-Time\": [\"1026\"],\n                                \"X-Ms-Operation-Identifier\": [\n                                    \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/uksouth/5c5c915c-65f2-46d6-a279-f918cf82b3b5\",\n                                ],\n                                \"X-Ms-Ratelimit-Remaining-Subscription-Global-Del...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036825136069349376","occurred_at":"2026-03-25T15:18:09Z","signature_id":"c7ab8c1e1837e1f667a77315ff2bf67e118cc09d1067356361cca699b15b4c3e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"56ae426faac218504147bb3e753625045b35215f15f2a9e3160aa9d64c9edeb1","failure_pattern":"found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-fdfcnq","search_query":"occurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-19..2026-03-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:245]: Unexpected error:\n    \u003c*errors.joinError | 0xc000d9d878\u003e: \n    found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-fdfcnq\n    {\n        errs: [\n            \u003c*errors.errorString | 0xc000718bc0\u003e{\n                s: \"found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-fdfcnq\",\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:245]: Unexpected error:\n    \u003c*errors.joinError | 0xc000d9d878\u003e: \n    found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-fdfcnq\n    {\n        errs: [\n            \u003c*errors.errorString | 0xc000718bc0\u003e{\n                s: \"found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-fdfcnq\",\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2036825136069349376","occurred_at":"2026-03-25T15:18:09Z","signature_id":"a229a2ed10a422e5dcbe43fdb34b2d1e12e80005ed6fb259f97886cd9eff408a","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}