{"meta":{"date":"2026-03-13","timezone":"UTC","generated_at":"2026-11-01T00:08:40Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":64,"failed_runs":21,"runs_with_occurrences":21,"runs_with_failure_pattern_matches":14,"runs_with_unmatched_occurrences":7,"runs_without_occurrence_rows":43,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4460/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032605796327493632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4460,"pr_state":"closed","pr_sha":"ab835e159c57f36bd2101ad503d287c62b978549","final_merged_sha":"ab835e159c57f36bd2101ad503d287c62b978549","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T23:52:40Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4447/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032602976715018240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4447,"pr_state":"closed","pr_sha":"3ef05c4706062e1df26f34038382392c0d192b35","final_merged_sha":"3ef05c4706062e1df26f34038382392c0d192b35","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T23:41:28Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032599807075291136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4457,"pr_state":"open","pr_sha":"0491181735904ceebcd6a6359d256fda98ad25c4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-13T23:28:52Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"4e7aeb1758cda570b144edbf79d05418723938bfb06b5c86d67ca2b0e8280274","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032599807075291136","occurred_at":"2026-03-13T23:28:52Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4459/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032596365107793920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4459,"pr_state":"closed","pr_sha":"40e4cea7f9c05cd0b1bdd436a7537118adb823e6","final_merged_sha":"40e4cea7f9c05cd0b1bdd436a7537118adb823e6","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T23:15:11Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032587564820467712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4429,"pr_state":"closed","pr_sha":"ff3b13945e68a754b201b7aa8ca83c486c517062","final_merged_sha":"ff3b13945e68a754b201b7aa8ca83c486c517062","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T22:40:13Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032559235597864960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4429,"pr_state":"closed","pr_sha":"ff3b13945e68a754b201b7aa8ca83c486c517062","final_merged_sha":"ff3b13945e68a754b201b7aa8ca83c486c517062","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T20:47:39Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032557289944125440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4219,"pr_state":"closed","pr_sha":"c8926b8ff8e632c2a446e743641c7f2b0c5b7f57","final_merged_sha":"8550ece78eb685f5f0357a6937bf6b6e9f457543","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-13T20:39:55Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032555317937573888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4441,"pr_state":"closed","pr_sha":"d0134d2dafd2560ecf5f5370f7dddac7a9ef8251","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-13T20:32:05Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4447/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032551246132416512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4447,"pr_state":"closed","pr_sha":"3ef05c4706062e1df26f34038382392c0d192b35","final_merged_sha":"3ef05c4706062e1df26f34038382392c0d192b35","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-13T20:15:54Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"6200ce76e6361042416b004f2f306a8198da4b94e06d9c1a980a2c847c738e2e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4447/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032551246132416512","occurred_at":"2026-03-13T20:15:54Z","failed_at":"e2e","signature_id":"989cc9288f3daa23826b41414c236db727979b02d4fd8296737539001bc069e2","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a3ac30\u003e: \n failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-4pgbh8, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-4pgbh8\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-4pgbh8, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-4pgbh8, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-4pgbh8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-4pgbh8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001224c50\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-4pgbh8\",\n },\n \u003c*fmt.wrapErrors | 0xc000a3ac00\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-4pgbh8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-4pgbh8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001224c50\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-4pgbh8\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a3ac30\u003e: \n failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-4pgbh8, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-4pgbh8\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-4pgbh8, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-4pgbh8, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-4pgbh8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-4pgbh8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001224c50\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-4pgbh8\",\n },\n \u003c*fmt.wrapErrors | 0xc000a3ac00\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-4pgbh8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-4pgbh8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001224c50\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-4pgbh8\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-wj68jn\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn, error: context deadline exceeded"}},{"row_id":"8bf04dab76330162add609e31a7a5113cc341b43a79bcee09ef3db40a8d419d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4447/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032551246132416512","occurred_at":"2026-03-13T20:15:54Z","failed_at":"e2e","signature_id":"f1baddbd50463d2968a950f7f352aa9d713f019c98a647fe2a7729a3f4f55f68","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:89]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000d69350\u003e: \n failed to create HCP cluster negative-tests-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-vdvmbz, error: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-vdvmbz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-vdvmbz, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster negative-tests-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-vdvmbz, error: failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-vdvmbz\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-vdvmbz, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00159cab0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-vdvmbz\",\n },\n \u003c*fmt.wrapErrors | 0xc000d69320\u003e{\n msg: \"failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-vdvmbz\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-vdvmbz, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00159cab0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-vdvmbz\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:89]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000d69350\u003e: \n failed to create HCP cluster negative-tests-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-vdvmbz, error: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-vdvmbz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-vdvmbz, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster negative-tests-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-vdvmbz, error: failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-vdvmbz\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-vdvmbz, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00159cab0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-vdvmbz\",\n },\n \u003c*fmt.wrapErrors | 0xc000d69320\u003e{\n msg: \"failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-vdvmbz\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-vdvmbz, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00159cab0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-vdvmbz\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-wj68jn\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn, error: context deadline exceeded"}},{"row_id":"ef97d1ba2e92954ea8d6b183470ebb30103aaac21f4aef3c05b451d1c48caef3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4447/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032551246132416512","occurred_at":"2026-03-13T20:15:54Z","failed_at":"e2e","signature_id":"418b0c5463613abb7754b824c41bbe2ac35c9318cd45c021487cb2208e8c4ed0","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a345d0\u003e: \n failed to create HCP cluster list-test-cluster-fg7g2l, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-fg7g2l in resource group cluster-listing-nc7jbv, error: failed waiting for cluster=\"list-test-cluster-fg7g2l\" in resourcegroup=\"cluster-listing-nc7jbv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-fg7g2l in resource group cluster-listing-nc7jbv, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster list-test-cluster-fg7g2l, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-fg7g2l in resource group cluster-listing-nc7jbv, error: failed waiting for cluster=\\\"list-test-cluster-fg7g2l\\\" in resourcegroup=\\\"cluster-listing-nc7jbv\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-fg7g2l in resource group cluster-listing-nc7jbv, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ea76a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-fg7g2l in resource group cluster-listing-nc7jbv\",\n },\n \u003c*fmt.wrapErrors | 0xc000a345a0\u003e{\n msg: \"failed waiting for cluster=\\\"list-test-cluster-fg7g2l\\\" in resourcegroup=\\\"cluster-listing-nc7jbv\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-fg7g2l in resource group cluster-listing-nc7jbv, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ea76a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-fg7g2l in resource group cluster-listing-nc7jbv\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a345d0\u003e: \n failed to create HCP cluster list-test-cluster-fg7g2l, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-fg7g2l in resource group cluster-listing-nc7jbv, error: failed waiting for cluster=\"list-test-cluster-fg7g2l\" in resourcegroup=\"cluster-listing-nc7jbv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-fg7g2l in resource group cluster-listing-nc7jbv, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster list-test-cluster-fg7g2l, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-fg7g2l in resource group cluster-listing-nc7jbv, error: failed waiting for cluster=\\\"list-test-cluster-fg7g2l\\\" in resourcegroup=\\\"cluster-listing-nc7jbv\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-fg7g2l in resource group cluster-listing-nc7jbv, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ea76a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-fg7g2l in resource group cluster-listing-nc7jbv\",\n },\n \u003c*fmt.wrapErrors | 0xc000a345a0\u003e{\n msg: \"failed waiting for cluster=\\\"list-test-cluster-fg7g2l\\\" in resourcegroup=\\\"cluster-listing-nc7jbv\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-fg7g2l in resource group cluster-listing-nc7jbv, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ea76a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-fg7g2l in resource group cluster-listing-nc7jbv\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-wj68jn\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn, error: context deadline exceeded"}},{"row_id":"f5586155eada4b7c19523193a62c784071a6821439434b2214b5e075dfad0dd1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4447/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032551246132416512","occurred_at":"2026-03-13T20:15:54Z","failed_at":"e2e","signature_id":"2d6bd47c81efcf4ade701c362730df716d3290a8ed7cf00af58857dbb0aced9b","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:90]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00173f4d0\u003e: \n failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-wj68jn\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn, error: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-wj68jn\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e03920\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn\",\n },\n \u003c*fmt.wrapErrors | 0xc00173f4a0\u003e{\n msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-wj68jn\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e03920\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:90]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00173f4d0\u003e: \n failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-wj68jn\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn, error: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-wj68jn\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e03920\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn\",\n },\n \u003c*fmt.wrapErrors | 0xc00173f4a0\u003e{\n msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-wj68jn\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e03920\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-wj68jn\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn, error: context deadline exceeded"}},{"row_id":"faa78a4f4c2d6563d8917b3a030dc60bbdff3c3a566f2a9894ffed015272ea51","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4447/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032551246132416512","occurred_at":"2026-03-13T20:15:54Z","failed_at":"e2e","signature_id":"dca9cc6e437f8e5ad847fdc6eaca7bedb56896e82d2c3d6758c2cb84d1d67aed","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a67590\u003e: \n failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-sv787q, error: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-sv787q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-sv787q, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-sv787q, error: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-sv787q\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-sv787q, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00111ba20\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-sv787q\",\n },\n \u003c*fmt.wrapErrors | 0xc000a67560\u003e{\n msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-sv787q\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-sv787q, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00111ba20\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-sv787q\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a67590\u003e: \n failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-sv787q, error: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-sv787q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-sv787q, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-sv787q, error: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-sv787q\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-sv787q, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00111ba20\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-sv787q\",\n },\n \u003c*fmt.wrapErrors | 0xc000a67560\u003e{\n msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-sv787q\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-sv787q, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00111ba20\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-sv787q\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-wj68jn\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4456/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032529632300896256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4456,"pr_state":"closed","pr_sha":"15a6607b96da75536dc6f08208e2837d2a222528","final_merged_sha":"15a6607b96da75536dc6f08208e2837d2a222528","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T18:49:10Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032527646637690880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4451,"pr_state":"closed","pr_sha":"afb2fcefc9674b612762533e592331eaa9dfbb68","final_merged_sha":"afb2fcefc9674b612762533e592331eaa9dfbb68","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T18:42:06Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4449/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032523475972788224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4449,"pr_state":"closed","pr_sha":"263454f69ae7f803865c16be87050ace7edda629","final_merged_sha":"205984e6ef0d1b76b1e624468408854ec2a65a8c","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-13T18:25:33Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4453/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032520172249550848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4453,"pr_state":"closed","pr_sha":"74066469c7ba25be3efcb17e4e6a47885ab78793","final_merged_sha":"74066469c7ba25be3efcb17e4e6a47885ab78793","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T18:12:26Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4281/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032520042368733184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4281,"pr_state":"open","pr_sha":"bfb41da5c2ddd65b821b95125c238a218d9447b0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-13T18:11:55Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032517673560051712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4454,"pr_state":"closed","pr_sha":"38f866ad42f9613c0978385116cd5b0c6df796c1","final_merged_sha":"38f866ad42f9613c0978385116cd5b0c6df796c1","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T18:02:30Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032504432188461056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4429,"pr_state":"closed","pr_sha":"ff3b13945e68a754b201b7aa8ca83c486c517062","final_merged_sha":"ff3b13945e68a754b201b7aa8ca83c486c517062","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T17:09:53Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4447/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032502223019184128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4447,"pr_state":"closed","pr_sha":"3ef05c4706062e1df26f34038382392c0d192b35","final_merged_sha":"3ef05c4706062e1df26f34038382392c0d192b35","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T17:00:52Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032498253500518400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4451,"pr_state":"closed","pr_sha":"afb2fcefc9674b612762533e592331eaa9dfbb68","final_merged_sha":"afb2fcefc9674b612762533e592331eaa9dfbb68","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T16:45:20Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4450/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032487840436195328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4450,"pr_state":"closed","pr_sha":"cb1d0d4acb67a0b6c1fd3be06011893b5e4b7b83","final_merged_sha":"cb1d0d4acb67a0b6c1fd3be06011893b5e4b7b83","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T16:03:57Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032481426963697664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4219,"pr_state":"closed","pr_sha":"c8926b8ff8e632c2a446e743641c7f2b0c5b7f57","final_merged_sha":"8550ece78eb685f5f0357a6937bf6b6e9f457543","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-13T15:38:27Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032480762573361152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4219,"pr_state":"closed","pr_sha":"c8926b8ff8e632c2a446e743641c7f2b0c5b7f57","final_merged_sha":"8550ece78eb685f5f0357a6937bf6b6e9f457543","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-13T15:35:38Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"ae5bcb94bf6ea7a03bf70de8679890f0403483e2a7555bc5fea97e5f75ed4e51","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032480762573361152","occurred_at":"2026-03-13T15:35:38Z","failed_at":"e2e","signature_id":"2922c3e95e9a31eb77fc4b5cab651ae19ecbe23d6d23e7e814e0c0de27eb589c","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00051fd80\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-ls6qjh\" to finish creating: GET https://rp.j3361152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e2649b12-f333-4a5a-9b0b-c671fe6c7f83\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e2649b12-f333-4a5a-9b0b-c671fe6c7f83\",\n \"name\": \"e2649b12-f333-4a5a-9b0b-c671fe6c7f83\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-13T16:20:16.794736263Z\",\n \"endTime\": \"2026-03-13T16:20:48.882954608Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to create transaction: pq: sorry, too many clients already\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-ls6qjh\\\" to finish creating: GET https://rp.j3361152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e2649b12-f333-4a5a-9b0b-c671fe6c7f83\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e2649b12-f333-4a5a-9b0b-c671fe6c7f83\\\",\\n \\\"name\\\": \\\"e2649b12-f333-4a5a-9b0b-c671fe6c7f83\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T16:20:16.794736263Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T16:20:48.882954608Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to create transaction: pq: sorry, too many clients already\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00051fcc0\u003e{\n msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-ls6qjh\\\" to finish creating: GET https://rp.j3361152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e2649b12-f333-4a5a-9b0b-c671fe6c7f83\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e2649b12-f333-4a5a-9b0b-c671fe6c7f83\\\",\\n \\\"name\\\": \\\"e2649b12-f333-4a5a-9b0b-c671fe6c7f83\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T16:20:16.794736263Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T16:20:48.882954608Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to create transaction: pq: sorry, too many clients already\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000d1cde0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Fri, 13 Mar 2026 16:20:57 GMT\",\n ],\n \"Content-Length\": [\"503\"],\n \"X-Envoy-Upstream-Service-Time\": [\"8\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"240ad451-7520-4429-832b-980707ba608c\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00051f840\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 9...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00051fd80\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-ls6qjh\" to finish creating: GET https://rp.j3361152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e2649b12-f333-4a5a-9b0b-c671fe6c7f83\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e2649b12-f333-4a5a-9b0b-c671fe6c7f83\",\n \"name\": \"e2649b12-f333-4a5a-9b0b-c671fe6c7f83\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-13T16:20:16.794736263Z\",\n \"endTime\": \"2026-03-13T16:20:48.882954608Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to create transaction: pq: sorry, too many clients already\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-ls6qjh\\\" to finish creating: GET https://rp.j3361152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e2649b12-f333-4a5a-9b0b-c671fe6c7f83\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e2649b12-f333-4a5a-9b0b-c671fe6c7f83\\\",\\n \\\"name\\\": \\\"e2649b12-f333-4a5a-9b0b-c671fe6c7f83\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T16:20:16.794736263Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T16:20:48.882954608Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to create transaction: pq: sorry, too many clients already\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00051fcc0\u003e{\n msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-ls6qjh\\\" to finish creating: GET https://rp.j3361152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e2649b12-f333-4a5a-9b0b-c671fe6c7f83\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e2649b12-f333-4a5a-9b0b-c671fe6c7f83\\\",\\n \\\"name\\\": \\\"e2649b12-f333-4a5a-9b0b-c671fe6c7f83\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T16:20:16.794736263Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T16:20:48.882954608Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to create transaction: pq: sorry, too many clients already\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000d1cde0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Fri, 13 Mar 2026 16:20:57 GMT\",\n ],\n \"Content-Length\": [\"503\"],\n \"X-Envoy-Upstream-Service-Time\": [\"8\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"240ad451-7520-4429-832b-980707ba608c\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00051f840\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 9...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"73f8eea4185fc2e89232d1d974d2ae534b494d3dac9a03f9838aa0bb8468e52f","failure_pattern":"ERROR CODE: InternalServerError; detail message failed to create transaction: pq: sorry, too many clients already; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["73f8eea4185fc2e89232d1d974d2ae534b494d3dac9a03f9838aa0bb8468e52f"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032479171392835584","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4347,"pr_state":"closed","pr_sha":"a82c62541bf569a06efb0a6211c6ae9e8a2c01a4","final_merged_sha":"a82c62541bf569a06efb0a6211c6ae9e8a2c01a4","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-13T15:29:30Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"02850b4ccdf6422bbb50fed2b38db1452e83c0e72eceac4dd75289759cf82724","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032479171392835584","occurred_at":"2026-03-13T15:29:30Z","failed_at":"e2e","signature_id":"9c1770c885f9ea79681774a9ece28b97d96517ef77afc34f7bf9e027dfafe816","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0013e57a0\u003e: \n failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-64dcpc, error: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-64dcpc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-64dcpc, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-64dcpc, error: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-64dcpc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-64dcpc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001126310\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-64dcpc\",\n },\n \u003c*fmt.wrapErrors | 0xc0013e5770\u003e{\n msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-64dcpc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-64dcpc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001126310\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-64dcpc\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0013e57a0\u003e: \n failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-64dcpc, error: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-64dcpc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-64dcpc, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-64dcpc, error: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-64dcpc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-64dcpc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001126310\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-64dcpc\",\n },\n \u003c*fmt.wrapErrors | 0xc0013e5770\u003e{\n msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-64dcpc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-64dcpc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001126310\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-64dcpc\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-wj68jn\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn, error: context deadline exceeded"}},{"row_id":"8776f5972f6849f1aaa6722ceb3a94f2f81bb127b4ace793cb51e3d8d9550f6c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032479171392835584","occurred_at":"2026-03-13T15:29:30Z","failed_at":"e2e","signature_id":"794ddf042d23f4d6f6be1fe742595bebed7eadedfab626a218278195ef603421","test_name":"Engineering should be able to retrieve expected metrics from the /metrics endpoint","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:74]: Failed to set up port-forward\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006c53e0\u003e: \n port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error\n {\n msg: \"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error\",\n err: \u003c*errors.errorString | 0xc00090cc20\u003e{\n s: \"error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:74]: Failed to set up port-forward\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006c53e0\u003e: \n port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error\n {\n msg: \"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error\",\n err: \u003c*errors.errorString | 0xc00090cc20\u003e{\n s: \"error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error\",\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f27efe24b36721c3037189b0270b3c037f3dcd8e282db885731d7395d97e1e2a","failure_pattern":"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing \u003cip\u003e:\u003cport\u003e, code 500: 500 Internal Server Error","search_query":"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error"}},{"row_id":"9f811712085ab46be9d1af5c93ca149dd133e683c2b74b1dabd79556658457a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032479171392835584","occurred_at":"2026-03-13T15:29:30Z","failed_at":"e2e","signature_id":"121aa69c6d76427609653b21cf483ba7ad17d01f2d19b5ea9866de12262cbad1","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e5ba40\u003e: \n failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-2k9wp9, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-2k9wp9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-2k9wp9, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-2k9wp9, error: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-2k9wp9\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-2k9wp9, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012377c0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-2k9wp9\",\n },\n \u003c*fmt.wrapErrors | 0xc000e5ba10\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-2k9wp9\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-2k9wp9, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012377c0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-2k9wp9\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e5ba40\u003e: \n failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-2k9wp9, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-2k9wp9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-2k9wp9, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-2k9wp9, error: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-2k9wp9\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-2k9wp9, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012377c0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-2k9wp9\",\n },\n \u003c*fmt.wrapErrors | 0xc000e5ba10\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-2k9wp9\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-2k9wp9, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012377c0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-2k9wp9\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-wj68jn\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-wj68jn, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","f27efe24b36721c3037189b0270b3c037f3dcd8e282db885731d7395d97e1e2a"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4432/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032478908145733632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4432,"pr_state":"closed","pr_sha":"b1ab51614479a0dd3e1236b827fc4ff1f09851f1","final_merged_sha":"b1ab51614479a0dd3e1236b827fc4ff1f09851f1","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T15:28:27Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4449/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032472799162929152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4449,"pr_state":"closed","pr_sha":"f0321d978a69241b0a16cda3ab612daa72c0293a","final_merged_sha":"205984e6ef0d1b76b1e624468408854ec2a65a8c","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-13T15:04:11Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4241/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032470482145513472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4241,"pr_state":"closed","pr_sha":"f0eb454d49936e73b94a7710cef63a611155b521","final_merged_sha":"f0eb454d49936e73b94a7710cef63a611155b521","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T14:54:58Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4435/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032469168887959552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4435,"pr_state":"closed","pr_sha":"4e77cea793cac38165cfba539cde69b73964b115","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-13T14:49:45Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"0113bb8fdacf69ef815b14a02db71e1075478af945f1ba57b9845dbfce4ff89f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4435/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032469168887959552","occurred_at":"2026-03-13T14:49:45Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032464923493142528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4224,"pr_state":"closed","pr_sha":"ff8e10ac2054752c79e8b466816be691e9c78d91","final_merged_sha":"0d2c310eee81cd65f33fd842930856c19dc160e3","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-13T14:32:53Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4281/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032461971307106304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4281,"pr_state":"open","pr_sha":"27be5dc07d694e2063574b29022d797354ac811e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-13T14:21:09Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4350/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032458648688529408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4350,"pr_state":"closed","pr_sha":"c117b0a88ebb40e9ee7f4bec2c1680e806014b69","final_merged_sha":"c117b0a88ebb40e9ee7f4bec2c1680e806014b69","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T14:07:57Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4384/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032453410116079616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4384,"pr_state":"closed","pr_sha":"76945bffc05e0be9c6747a6366cbd4377839d85e","final_merged_sha":"76945bffc05e0be9c6747a6366cbd4377839d85e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T13:46:16Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4448/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032448848714010624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4448,"pr_state":"closed","pr_sha":"0305ab92d286e4c35bbf5f1d82bef5ecd5a51c65","final_merged_sha":"0305ab92d286e4c35bbf5f1d82bef5ecd5a51c65","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T13:29:01Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4447/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032443105361268736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4447,"pr_state":"closed","pr_sha":"6bb5230701c523a5fd8146cabf710f6137a89381","final_merged_sha":"3ef05c4706062e1df26f34038382392c0d192b35","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-13T13:06:11Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"1f717c33fb0c777c7891fb02f9f0ba4eec4175fa52609cb23e2d13b59b808576","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4447/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032443105361268736","occurred_at":"2026-03-13T13:06:11Z","failed_at":"e2e","signature_id":"b337d17575816b7583e79a0f3a84cae5b9d24a2499840b32b2503ed208cb7a96","test_name":"4.y-stream Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade to the possible candidate release upgrade if available","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:186]: got 2 error(s) from parallel 4.y runs: [4.21: VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.3) failed: no version in clusterversion/version status.history is greater than initial \"4.21.3\" 4.20: VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \"4.20.15\"]\nExpected\n \u003c[]error | len:2, cap:2\u003e: [\n \u003c*fmt.wrapError | 0xc000ad8040\u003e{\n msg: \"4.21: VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.3) failed: no version in clusterversion/version status.history is greater than initial \\\"4.21.3\\\"\",\n err: \u003c*errors.joinError | 0xc0011cef00\u003e{\n errs: [\n \u003c*fmt.wrapError | 0xc000ad8020\u003e{\n msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.3) failed: no version in clusterversion/version status.history is greater than initial \\\"4.21.3\\\"\",\n err: \u003c*errors.errorString | 0xc000e42e50\u003e{\n s: \"no version in clusterversion/version status.history is greater than initial \\\"4.21.3\\\"\",\n },\n },\n ],\n },\n },\n \u003c*fmt.wrapError | 0xc000f08220\u003e{\n msg: \"4.20: VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n err: \u003c*errors.joinError | 0xc00146eba0\u003e{\n errs: [\n \u003c*fmt.wrapError | 0xc000f081e0\u003e{\n msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n err: \u003c*errors.errorString | 0xc0012e51f0\u003e{\n s: \"no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n },\n },\n ],\n },\n },\n ]\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:186]: got 2 error(s) from parallel 4.y runs: [4.21: VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.3) failed: no version in clusterversion/version status.history is greater than initial \"4.21.3\" 4.20: VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \"4.20.15\"]\nExpected\n \u003c[]error | len:2, cap:2\u003e: [\n \u003c*fmt.wrapError | 0xc000ad8040\u003e{\n msg: \"4.21: VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.3) failed: no version in clusterversion/version status.history is greater than initial \\\"4.21.3\\\"\",\n err: \u003c*errors.joinError | 0xc0011cef00\u003e{\n errs: [\n \u003c*fmt.wrapError | 0xc000ad8020\u003e{\n msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.3) failed: no version in clusterversion/version status.history is greater than initial \\\"4.21.3\\\"\",\n err: \u003c*errors.errorString | 0xc000e42e50\u003e{\n s: \"no version in clusterversion/version status.history is greater than initial \\\"4.21.3\\\"\",\n },\n },\n ],\n },\n },\n \u003c*fmt.wrapError | 0xc000f08220\u003e{\n msg: \"4.20: VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n err: \u003c*errors.joinError | 0xc00146eba0\u003e{\n errs: [\n \u003c*fmt.wrapError | 0xc000f081e0\u003e{\n msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n err: \u003c*errors.errorString | 0xc0012e51f0\u003e{\n s: \"no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n },\n },\n ],\n },\n },\n ]\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4f56dc5a0ca687ae8a853bff15f8595db6dff13c5a7a68892ee35b534b0e8030","failure_pattern":"got 2 error(s) from parallel 4.y runs: [4.21: VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.3) failed: no version in clusterversion/version status.history is greater than initial \"4.21.3\" 4.20","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:186]: got 2 error(s) from parallel 4.y runs: [4.21: VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.3) failed: no version in clusterversion/version status.history is greater than initial \"4.21.3\" 4.20: VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \"4.20.15\"]"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4f56dc5a0ca687ae8a853bff15f8595db6dff13c5a7a68892ee35b534b0e8030"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4435/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032441906725654528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4435,"pr_state":"closed","pr_sha":"f744b064a4c3374f130563c3ec284889a1459a02","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-13T13:01:25Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"7503d138104132f7ceb9a2fefe9fccf9ec26c64e94b1f5ed1f725c12917c0d15","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4435/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032441906725654528","occurred_at":"2026-03-13T13:01:25Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4350/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032441026731315200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4350,"pr_state":"closed","pr_sha":"c117b0a88ebb40e9ee7f4bec2c1680e806014b69","final_merged_sha":"c117b0a88ebb40e9ee7f4bec2c1680e806014b69","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-13T12:57:56Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"6bc311d6172306db0bfd30f8acfe2021ff5892b2a6a2c854055df8091c28bb89","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4350/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032441026731315200","occurred_at":"2026-03-13T12:57:56Z","failed_at":"provision","signature_id":"1e3c66f1b3aaa8e9531a0447e5fe758d4cfa54cfe1d96c0e23b03169e36088b8","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","test_suite":"step graph","failure_text":"time=2026-03-13T13:34:35.658Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n Kind: Shell\\n Command: ./istio.sh\\n\"\ntime=2026-03-13T13:34:38.234Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"error running Shell Step, failed to execute shell command: + set -euo pipefail\\n+ ISTIO_NAMESPACE=aks-istio-system\\n+ echo '********** Check istio is up and running **************'\\n********** Check istio is up and running **************\\n++ kubectl get pods -n aks-istio-system -l istio.io/rev=asm-1-26 --field-selector=status.phase=Running --no-headers\\n++ wc -l\\n+ ISTIO_PODS_COUNT=2\\n+ [[ 2 -lt 2 ]]\\n+ echo '********** Download istioctl **************'\\n********** Download istioctl **************\\n++ uname\\n+ OS=Linux\\n+ '[' Linux = Darwin ']'\\n+ OSEXT=linux\\n++ uname -m\\n+ LOCAL_ARCH=x86_64\\n+ case \\\"${LOCAL_ARCH}\\\" in\\n+ ISTIO_ARCH=amd64\\n+ ISTIO_URL=https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz\\n+ SHA256_URL=https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz.sha256\\n+ curl -sL https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz -o istio-1.28.3-linux-amd64.tar.gz\\n+ curl -sL https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz.sha256 -o istio-1.28.3-linux-amd64.tar.gz.sha256\\n+ sha256sum -c istio-1.28.3-linux-amd64.tar.gz.sha256\\nistio-1.28.3-linux-amd64.tar.gz: FAILED\\nsha256sum: WARNING: 1 computed checksum did NOT match\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"941f12df2ac8109e85b745c88ba5fd5783296c619a1efe2cf51485b6dc6bc270","failure_pattern":"istio-1.28.3-linux-amd64.tar.gz: FAILED","search_query":"istio-1.28.3-linux-amd64.tar.gz: FAILED"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["941f12df2ac8109e85b745c88ba5fd5783296c619a1efe2cf51485b6dc6bc270"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4447/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032437173365510144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4447,"pr_state":"closed","pr_sha":"6bb5230701c523a5fd8146cabf710f6137a89381","final_merged_sha":"3ef05c4706062e1df26f34038382392c0d192b35","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-13T12:42:37Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"1f54a8cdc5b27e7e8afcbfd2ba7c7c9214a9458db5fbd701cb42aeb0bb12fd9d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4447/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032437173365510144","occurred_at":"2026-03-13T12:42:37Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4446/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032436896004575232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4446,"pr_state":"open","pr_sha":"5ed9c72a6262f923de3a314d902f2b394fd3765b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-13T12:41:30Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4442/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032435565269356544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4442,"pr_state":"closed","pr_sha":"8ac2a500d7cade2ccbab11984c86c49cd0e20d4f","final_merged_sha":"8ac2a500d7cade2ccbab11984c86c49cd0e20d4f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T12:36:14Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4435/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032432480593645568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4435,"pr_state":"closed","pr_sha":"f9c9bd7696d5b3b8efb8c81b427add149a153b9c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-13T12:23:58Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"fd96a65b620b99d3b249afec0b10fa43ded6d5dabc96144f20edc6a4afc28c02","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4435/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032432480593645568","occurred_at":"2026-03-13T12:23:58Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032423613738520576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4224,"pr_state":"closed","pr_sha":"71882615df33e7a9266d223ac9eec39d31fc7de1","final_merged_sha":"0d2c310eee81cd65f33fd842930856c19dc160e3","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-13T11:48:44Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"8fad3413343e308c284eb78381e664760bca7e394c858296f613bbd80c5ca7d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032423613738520576","occurred_at":"2026-03-13T11:48:44Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4241/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032420331267821568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4241,"pr_state":"closed","pr_sha":"b3596057415d400b08c668123e62dc03b2282d9b","final_merged_sha":"f0eb454d49936e73b94a7710cef63a611155b521","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-13T11:35:42Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4281/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032419297984253952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4281,"pr_state":"open","pr_sha":"c7084430c080e6b51bb08c12c44c9fb63914f9e6","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-13T11:30:57Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4438/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032410750961586176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4438,"pr_state":"closed","pr_sha":"45a701a3efe2ef6e5ba12c36735594cc9c88d1e4","final_merged_sha":"45a701a3efe2ef6e5ba12c36735594cc9c88d1e4","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T10:57:37Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4385/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032409426824007680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4385,"pr_state":"closed","pr_sha":"07e421bae19659fd95e6d16899bed8ad2fda3863","final_merged_sha":"07e421bae19659fd95e6d16899bed8ad2fda3863","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T10:52:22Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032404943163363328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4429,"pr_state":"closed","pr_sha":"e9aaf26410cc0d0c55724c8b7e47cb72b4ecb200","final_merged_sha":"ff3b13945e68a754b201b7aa8ca83c486c517062","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-13T10:34:33Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4443/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032391539950358528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4443,"pr_state":"closed","pr_sha":"53a705301c2acfc2a39dcba371fa99ace1ffd433","final_merged_sha":"53a705301c2acfc2a39dcba371fa99ace1ffd433","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T09:41:17Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4432/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032386363810123776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4432,"pr_state":"closed","pr_sha":"b1ab51614479a0dd3e1236b827fc4ff1f09851f1","final_merged_sha":"b1ab51614479a0dd3e1236b827fc4ff1f09851f1","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-13T09:20:43Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3c6c671db9e3304bce00ba86a3a63e066fe85204bf7f31c678f8552da7b7462e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4432/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032386363810123776","occurred_at":"2026-03-13T09:20:43Z","failed_at":"e2e","signature_id":"1a61e87a825c5825b71f015eaebc592ff491187886c00286edfb5a41efe36d02","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0009acde0\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-55ftph\" to finish creating: GET https://rp.j0123776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ae023f88-78a9-4f67-80c7-208e1aa60e21\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ae023f88-78a9-4f67-80c7-208e1aa60e21\",\n \"name\": \"ae023f88-78a9-4f67-80c7-208e1aa60e21\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-13T10:07:57.515747627Z\",\n \"endTime\": \"2026-03-13T10:08:36.976860169Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: 429\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"429\\\",\\n \\\"message\\\": \\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-55ftph\\\" to finish creating: GET https://rp.j0123776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ae023f88-78a9-4f67-80c7-208e1aa60e21\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ae023f88-78a9-4f67-80c7-208e1aa60e21\\\",\\n \\\"name\\\": \\\"ae023f88-78a9-4f67-80c7-208e1aa60e21\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T10:07:57.515747627Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T10:08:36.976860169Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: 429\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"429\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0009acdc0\u003e{\n msg: \"failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-55ftph\\\" to finish creating: GET https://rp.j0123776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ae023f88-78a9-4f67-80c7-208e1aa60e21\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ae023f88-78a9-4f67-80c7-208e1aa60e21\\\",\\n \\\"name\\\": \\\"ae023f88-78a9-4f67-80c7-208e1aa60e21\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T10:07:57.515747627Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T10:08:36.976860169Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: 429\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"429\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too man...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0009acde0\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-55ftph\" to finish creating: GET https://rp.j0123776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ae023f88-78a9-4f67-80c7-208e1aa60e21\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ae023f88-78a9-4f67-80c7-208e1aa60e21\",\n \"name\": \"ae023f88-78a9-4f67-80c7-208e1aa60e21\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-13T10:07:57.515747627Z\",\n \"endTime\": \"2026-03-13T10:08:36.976860169Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: 429\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"429\\\",\\n \\\"message\\\": \\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-55ftph\\\" to finish creating: GET https://rp.j0123776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ae023f88-78a9-4f67-80c7-208e1aa60e21\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ae023f88-78a9-4f67-80c7-208e1aa60e21\\\",\\n \\\"name\\\": \\\"ae023f88-78a9-4f67-80c7-208e1aa60e21\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T10:07:57.515747627Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T10:08:36.976860169Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: 429\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"429\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0009acdc0\u003e{\n msg: \"failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-55ftph\\\" to finish creating: GET https://rp.j0123776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ae023f88-78a9-4f67-80c7-208e1aa60e21\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ae023f88-78a9-4f67-80c7-208e1aa60e21\\\",\\n \\\"name\\\": \\\"ae023f88-78a9-4f67-80c7-208e1aa60e21\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T10:07:57.515747627Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T10:08:36.976860169Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: 429\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"429\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too man...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4ea33acd3f1140f6ce3e5704a686a67d6df1f6b9e581d4685b990aa2749d6360","failure_pattern":"ERROR CODE: InternalServerError; detail code 429; detail message Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4ea33acd3f1140f6ce3e5704a686a67d6df1f6b9e581d4685b990aa2749d6360"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032385424185364480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4224,"pr_state":"closed","pr_sha":"379c5f5722666be859fa3dc518647c310b440fd4","final_merged_sha":"0d2c310eee81cd65f33fd842930856c19dc160e3","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-13T09:16:59Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"4e0a720edaf83485549cd0fba4adc715e34ffa9918331ce8384b5baddda80f15","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032385424185364480","occurred_at":"2026-03-13T09:16:59Z","failed_at":"e2e","signature_id":"8b87fad834c85b47b73172701d4dd8ea9c10dca77406f2355a3b33cda1f6cb1e","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00081a320\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-g8vnjb\" to finish creating: GET https://rp.j5364480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2dba5912-02d4-4f25-9317-8f3c4864921e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2dba5912-02d4-4f25-9317-8f3c4864921e\",\n \"name\": \"2dba5912-02d4-4f25-9317-8f3c4864921e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-13T10:03:46.091065162Z\",\n \"endTime\": \"2026-03-13T10:04:45.355605211Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Failed to update operator file-csi-driver managed identity with client ID and principal ID in cluster 2p0p60dcjbncn6sutal35r1u1h5mu6vm: failed to create transaction: pq: sorry, too many clients already\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-g8vnjb\\\" to finish creating: GET https://rp.j5364480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2dba5912-02d4-4f25-9317-8f3c4864921e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2dba5912-02d4-4f25-9317-8f3c4864921e\\\",\\n \\\"name\\\": \\\"2dba5912-02d4-4f25-9317-8f3c4864921e\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T10:03:46.091065162Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T10:04:45.355605211Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to update operator file-csi-driver managed identity with client ID and principal ID in cluster 2p0p60dcjbncn6sutal35r1u1h5mu6vm: failed to create transaction: pq: sorry, too many clients already\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00081a300\u003e{\n msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-g8vnjb\\\" to finish creating: GET https://rp.j5364480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2dba5912-02d4-4f25-9317-8f3c4864921e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2dba5912-02d4-4f25-9317-8f3c4864921e\\\",\\n \\\"name\\\": \\\"2dba5912-02d4-4f25-9317-8f3c4864921e\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T10:03:46.091065162Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T10:04:45.355605211Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to update operator file-csi-driver managed identity with client ID and principal ID in cluster 2p0p60dcjbncn6sutal35r1u1h5mu6vm: failed to create transaction: pq: sorry, too many clients already\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012106c0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"24445649-3067-4512-ab56-30450f25a3c5\",\n ],\n \"Date\": [\n \"Fri, 13 Mar 2026 10:04:46 GMT\",\n ],\n \"Content-Length\": [\"639\"],\n \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00081a200\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 1...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00081a320\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-g8vnjb\" to finish creating: GET https://rp.j5364480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2dba5912-02d4-4f25-9317-8f3c4864921e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2dba5912-02d4-4f25-9317-8f3c4864921e\",\n \"name\": \"2dba5912-02d4-4f25-9317-8f3c4864921e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-13T10:03:46.091065162Z\",\n \"endTime\": \"2026-03-13T10:04:45.355605211Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Failed to update operator file-csi-driver managed identity with client ID and principal ID in cluster 2p0p60dcjbncn6sutal35r1u1h5mu6vm: failed to create transaction: pq: sorry, too many clients already\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-g8vnjb\\\" to finish creating: GET https://rp.j5364480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2dba5912-02d4-4f25-9317-8f3c4864921e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2dba5912-02d4-4f25-9317-8f3c4864921e\\\",\\n \\\"name\\\": \\\"2dba5912-02d4-4f25-9317-8f3c4864921e\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T10:03:46.091065162Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T10:04:45.355605211Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to update operator file-csi-driver managed identity with client ID and principal ID in cluster 2p0p60dcjbncn6sutal35r1u1h5mu6vm: failed to create transaction: pq: sorry, too many clients already\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00081a300\u003e{\n msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-g8vnjb\\\" to finish creating: GET https://rp.j5364480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2dba5912-02d4-4f25-9317-8f3c4864921e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2dba5912-02d4-4f25-9317-8f3c4864921e\\\",\\n \\\"name\\\": \\\"2dba5912-02d4-4f25-9317-8f3c4864921e\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T10:03:46.091065162Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T10:04:45.355605211Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to update operator file-csi-driver managed identity with client ID and principal ID in cluster 2p0p60dcjbncn6sutal35r1u1h5mu6vm: failed to create transaction: pq: sorry, too many clients already\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012106c0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"24445649-3067-4512-ab56-30450f25a3c5\",\n ],\n \"Date\": [\n \"Fri, 13 Mar 2026 10:04:46 GMT\",\n ],\n \"Content-Length\": [\"639\"],\n \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00081a200\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 1...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f40a1c6ee7996ef083f3791d136f99f0d6bec6f64192d9aa7899ab087d2bd422","failure_pattern":"ERROR CODE: InternalServerError; detail message Failed to update operator file-csi-driver managed identity with client ID and principal ID in cluster \u003ccluster\u003e: failed to create transaction: pq: sorry, too many clients already; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f40a1c6ee7996ef083f3791d136f99f0d6bec6f64192d9aa7899ab087d2bd422"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4384/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032382713658347520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4384,"pr_state":"closed","pr_sha":"76945bffc05e0be9c6747a6366cbd4377839d85e","final_merged_sha":"76945bffc05e0be9c6747a6366cbd4377839d85e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T09:06:13Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4442/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032378301669969920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4442,"pr_state":"closed","pr_sha":"bbf5b4dbc65b2ab2f1d2fee879432cb3010e1426","final_merged_sha":"8ac2a500d7cade2ccbab11984c86c49cd0e20d4f","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-13T08:48:41Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4385/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032377115084591104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4385,"pr_state":"closed","pr_sha":"07e421bae19659fd95e6d16899bed8ad2fda3863","final_merged_sha":"07e421bae19659fd95e6d16899bed8ad2fda3863","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-13T08:43:58Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d83a8ce4dbaaba1c19938ce908c70c541f467b606ea6b83727f08f08165bb98f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4385/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032377115084591104","occurred_at":"2026-03-13T08:43:58Z","failed_at":"e2e","signature_id":"54270933ced1d5ebb5bdb4f4f814fc26f0310a475883d2b6aaa896d5316e6a9a","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129]: Unexpected error:\n \u003c*exported.ResponseError | 0xc0013e27b0\u003e: \n GET https://rp.j4591104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d1533905-6dac-43f4-90c6-d9eaae459507\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d1533905-6dac-43f4-90c6-d9eaae459507\",\n \"name\": \"d1533905-6dac-43f4-90c6-d9eaae459507\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-13T09:37:46.323005448Z\",\n \"endTime\": \"2026-03-13T09:38:20.435738147Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-28/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-28/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"2d4bb247-b8c7-41e4-8de7-8ac18e5b0f46\",\n ],\n \"Date\": [\n \"Fri, 13 Mar 2026 09:38:26 GMT\",\n ],\n \"Content-Length\": [\"1539\"],\n \"X-Envoy-Upstream-Service-Time\": [\"9\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00063a2c0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 100, 49, 53, 51, 51, 57, 48, 53, 45, 54, 100, 97, 99, 45, 52, 51, 102, 52, 45, 57, 48, 99, 54, 45, 100, 57, 101, 97, 97, 101, 52, 53, 57, 53, 48, 55, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 100, 49, 53, 51, 51, 57, 48, 53, 45, 54, 100, 97, 99, 45, 52, 51, 102, 52, 45, 57, 48, 99, 54, 45, 100, 57, 101, 97, 97, 101, 52, 53, 57, 53, 48, 55, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 116, 117, 115, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 114, 116, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 51, 84, 48, 57, 58, 51, 55, 58, 52, 54, 46, 51, 50, 51, 48, 48, 53, 52, 52, 56, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 110, 100, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 51, 84, 48, 57, 58, 51, 56, 58, 50, 48, 46, 52, 51, 53, 55, 51, 56, 49, 52, 55, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 102, 97, 105, 108, 101, 100, 32, 116, 111, 32, 103, 101, 116, 32, 109, 97, 110, 97, 103, 101, 100, 32, 105, 100, 101, 110, 116, 105, 116, 121, 32, 39, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 97, 114, 111, 45, 104, 99, 112, 45, 116, 101, 115, 116, 45, 109, 115, 105, 45, 99, 111, 110, 116, 97, 105, 110, 101, 114, 115, 45, 100, 101, 118, 45, 50, 56, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 77, 97, 110, 97, 103, 101, 100, 73, 100, 101, 110, 116, 105, 116, 121, 47, 117, 115, 101, 114, 65, 115, 115, 105, 103, 110, 101, 100, 73, 100, 101, 110, 116, 105, 116, 105, 101, 115, 47, 107, 109, 115, 39, 58, 32, 71, 69, 84, 32, 104, 116, 116, 112, 115, 58, 47, 47, 109, 97, 110, 97, 103, 101, 109, 101, 110, 116, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 1...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129]: Unexpected error:\n \u003c*exported.ResponseError | 0xc0013e27b0\u003e: \n GET https://rp.j4591104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d1533905-6dac-43f4-90c6-d9eaae459507\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d1533905-6dac-43f4-90c6-d9eaae459507\",\n \"name\": \"d1533905-6dac-43f4-90c6-d9eaae459507\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-13T09:37:46.323005448Z\",\n \"endTime\": \"2026-03-13T09:38:20.435738147Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-28/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-28/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"2d4bb247-b8c7-41e4-8de7-8ac18e5b0f46\",\n ],\n \"Date\": [\n \"Fri, 13 Mar 2026 09:38:26 GMT\",\n ],\n \"Content-Length\": [\"1539\"],\n \"X-Envoy-Upstream-Service-Time\": [\"9\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00063a2c0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 100, 49, 53, 51, 51, 57, 48, 53, 45, 54, 100, 97, 99, 45, 52, 51, 102, 52, 45, 57, 48, 99, 54, 45, 100, 57, 101, 97, 97, 101, 52, 53, 57, 53, 48, 55, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 100, 49, 53, 51, 51, 57, 48, 53, 45, 54, 100, 97, 99, 45, 52, 51, 102, 52, 45, 57, 48, 99, 54, 45, 100, 57, 101, 97, 97, 101, 52, 53, 57, 53, 48, 55, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 116, 117, 115, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 114, 116, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 51, 84, 48, 57, 58, 51, 55, 58, 52, 54, 46, 51, 50, 51, 48, 48, 53, 52, 52, 56, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 110, 100, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 51, 84, 48, 57, 58, 51, 56, 58, 50, 48, 46, 52, 51, 53, 55, 51, 56, 49, 52, 55, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 102, 97, 105, 108, 101, 100, 32, 116, 111, 32, 103, 101, 116, 32, 109, 97, 110, 97, 103, 101, 100, 32, 105, 100, 101, 110, 116, 105, 116, 121, 32, 39, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 97, 114, 111, 45, 104, 99, 112, 45, 116, 101, 115, 116, 45, 109, 115, 105, 45, 99, 111, 110, 116, 97, 105, 110, 101, 114, 115, 45, 100, 101, 118, 45, 50, 56, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 77, 97, 110, 97, 103, 101, 100, 73, 100, 101, 110, 116, 105, 116, 121, 47, 117, 115, 101, 114, 65, 115, 115, 105, 103, 110, 101, 100, 73, 100, 101, 110, 116, 105, 116, 105, 101, 115, 47, 107, 109, 115, 39, 58, 32, 71, 69, 84, 32, 104, 116, 116, 112, 115, 58, 47, 47, 109, 97, 110, 97, 103, 101, 109, 101, 110, 116, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 1...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032350755284848640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4441,"pr_state":"closed","pr_sha":"d0134d2dafd2560ecf5f5370f7dddac7a9ef8251","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-13T06:58:54Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4431/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032344810727149568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4431,"pr_state":"closed","pr_sha":"73508c2fd876a44a5ce35c0026ec05ac1d7294b1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-13T06:35:36Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"8402f813f73989e71d0a0dc0acbc6b3273a601ba7d0c26744e13d32aab899f1e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4431/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032344810727149568","occurred_at":"2026-03-13T06:35:36Z","failed_at":"provision","signature_id":"2909b7ce1875a155f14a0c9986a8cdeb7cee739e35f88e948429a66183545b10","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Server/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-13T07:16:50.735Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-13T07:16:51.321Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy values=\"map[azure:map[clientId:70425aa6-afd8-4c6b-aad1-2b49f9f4fe42 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] backend:map[namespace:aro-hcp serviceAccount:backend] broker:map[host:arohcp-prow-maestro-j7149568.westus3-1.ts.eventgrid.azure.net port:8883] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:maestro-server-j7149568] database:map[azureDb:false containerizedDbImage:docker.io/library/postgres:14.2 containerizedDbPvcCapacity:512Mi debug:false host:maestro-db maxOpenConnections:50 name:maestro password:TheBlurstOfTimes port:5432 user:maestro-server] deployment:map[limits:map[cpu:1 memory:1Gi] replicas:3 requests:map[cpu:200m memory:512Mi] zoneCount:4] image:map[digest:sha256:6b1095324bbe9f6ffab3fd93de0bbc5b70d75b51545de50daa4f1d9f460ba7bf registry:arohcpsvcdev.azurecr.io repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro] istio:map[restrictIngress:true] maestro:map[glog_v:2 grpcBindPort:8090 healthCheckBindPort:8083 httpBindPort:8000 httpReadTimeout:60s httpWriteTimeout:60s labelMetricsInclusionDuration:168h serviceAccount:maestro] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro:pull workloadIdentityClientId:65514dfa-72eb-404e-bee6-b4614adf7122 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-13T07:16:51.321Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-13T07:16:51.562Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy namespace=maestro\ntime=2026-03-13T07:16:51.871Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy dryRun=false\ntime=2026-03-13T07:16:51.871Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-13T07:16:51.987Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-13T07:21:59.462Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy err=\"resource not ready, name: maestro, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-13T07:21:59.463Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-13T07:21:59.463Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-13T07:22:03.762Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy release=maestro-server namespace=maestro status=failed description=\"Release \\\"maestro-server\\\" failed: resource not ready, name: maestro, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[azure:map[clientId:70425aa6-afd8-4c6b-aad1-2b49f9f4fe42 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] backend:map[namespace:aro-hcp serviceAccount:backend] broker:map[host:arohcp-prow-maestro-j7149568.westus3-1.ts.eventgrid.azure.net port:8883] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:maestro-server-j7149568] database:map[azureDb:false containerizedDbImage:docker.io/library/postgres:14.2 containerizedDbPvcCapacity:512Mi debug:false host:maestro-db maxOpenConnections:50 name:maestro password:TheBlurstOfTimes port:5432 user:maestro-server] deployment:map[limits:map[cpu:1 memory:1Gi] replicas:3 requests:map[cpu:200m memory:512Mi] zoneCount:4] image:map[digest:sha256:6b1095324bbe9f6ffab3fd93de0bbc5b70d75b51545de50daa4f1d9f460ba7bf registry:arohcpsvcdev.azurecr.io repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro] istio:map[restrictIngress:true] maestro:map[glog_v:2 grpcBindPort:8090 healthCheckBindPort:8083 httpBindPort:8000 httpReadTimeout:60s httpWriteTimeout:60s labelMetricsInclusionDuration:168h serviceAccount:maestro] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro:pull workloadIdentityClientId:65514dfa-72eb-404e-bee6-b4614adf7122 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-13T07:22:03.770Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5yUQW/aTBCG7/4VK59AsiMgSj59VDmkaU9VK6RUPaTqYb371l5Y71izYxAVP74ypIEkpoEePX6eV7Ozq0k8RDEitWwQ1Y2yWrTowmNAxRxGPrlgp1HYhTJTu9IXXeOphCXCthIbbf6Uh98TpZRKbw3PWu/fu2BdKNNMpU3rfV7sv2uNKExp9ii0UhG7X1ochRl5Z9Ydpb2nVV5os0Cw52rGt1HAeQQvncG5+iObC+W2OFuGsDPxXC2QVP8wIEbpovD2Xz6nouv5yX8VNQO4i0MQZ7ZOB1n81K2XHv5+N7/PFJwQHwDbyYKPK7fGUBueZ/awhnEu038lM3B0URDkG/m2xp3Xrn7Tun/1PvKSG3MaWUF7qUwFszhJOC31+PPpgY8cqpvYjGnpLPjO6xj/3sUHNJ7WNd66rl6uv4fnaENRSkbMC4ZelC87Sn68SxZtAQ4QxI/ddonJRq0qMJS4GlF03agCsgKCGlgt6MqDdDKaXOejy3x8+XX033R8Pb0aP6RDdXGhjjGT8fTq/4d0OEw2ak4uqIUL9saFAD7YiRTUfhUe7sCXyy/ZqIap+7tvNFMN2R18WkimPJWZqhGjLrtIYgtWxfrg8BbR/A4AAP//X0LiUbkFAAA=\"\ntime=2026-03-13T07:22:03.777Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-13T07:22:03.778Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: maestro, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"79148d5b41854006220f3b0e4d75a35d8994ba27457408fb48cc3b344d2b4121","failure_pattern":"resource not ready, name: maestro, kind: Deployment, status: InProgress context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: maestro, kind: Deployment, status: InProgress\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["79148d5b41854006220f3b0e4d75a35d8994ba27457408fb48cc3b344d2b4121"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4385/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032336541686173696","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4385,"pr_state":"closed","pr_sha":"24eff95d3608501a0b1e004ee49e4f684315db44","final_merged_sha":"07e421bae19659fd95e6d16899bed8ad2fda3863","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-13T06:02:45Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032296842850471936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4399,"pr_state":"closed","pr_sha":"8a1ae2ac7a78b624556d264008ca37e8c812709f","final_merged_sha":"8a1ae2ac7a78b624556d264008ca37e8c812709f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T03:25:00Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032296150022754304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4399,"pr_state":"closed","pr_sha":"8a1ae2ac7a78b624556d264008ca37e8c812709f","final_merged_sha":"8a1ae2ac7a78b624556d264008ca37e8c812709f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T03:22:15Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4409/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032291092698763264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4409,"pr_state":"closed","pr_sha":"61fd73675b4e600ec10d87e4a1ef0ac193215b57","final_merged_sha":"61fd73675b4e600ec10d87e4a1ef0ac193215b57","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T03:02:04Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032268928834605056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4436,"pr_state":"closed","pr_sha":"126c1612f05ec23dc781c781eacc6707538b464a","final_merged_sha":"126c1612f05ec23dc781c781eacc6707538b464a","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T01:34:04Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032254932328910848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4436,"pr_state":"closed","pr_sha":"126c1612f05ec23dc781c781eacc6707538b464a","final_merged_sha":"126c1612f05ec23dc781c781eacc6707538b464a","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-13T00:38:23Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"5ccbea226483fc0c11cb5d5d71de1f45244a70ffb4a9a3900b9fa0dbd93edbce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032254932328910848","occurred_at":"2026-03-13T00:38:23Z","failed_at":"e2e","signature_id":"18639f059aed997e8709193fe02f1ac51042b4fe3bf8e47e50d065e4096c3370","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0008e25e0\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-rnjkhd\" to finish creating: GET https://rp.j8910848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b86a247-0f2a-4a31-bc1f-16f9237d39d6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b86a247-0f2a-4a31-bc1f-16f9237d39d6\",\n \"name\": \"0b86a247-0f2a-4a31-bc1f-16f9237d39d6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-13T01:28:43.185485913Z\",\n \"endTime\": \"2026-03-13T01:30:19.825677573Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-rnjkhd\\\" to finish creating: GET https://rp.j8910848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b86a247-0f2a-4a31-bc1f-16f9237d39d6\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b86a247-0f2a-4a31-bc1f-16f9237d39d6\\\",\\n \\\"name\\\": \\\"0b86a247-0f2a-4a31-bc1f-16f9237d39d6\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T01:28:43.185485913Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T01:30:19.825677573Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0008e25c0\u003e{\n msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-rnjkhd\\\" to finish creating: GET https://rp.j8910848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b86a247-0f2a-4a31-bc1f-16f9237d39d6\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b86a247-0f2a-4a31-bc1f-16f9237d39d6\\\",\\n \\\"name\\\": \\\"0b86a247-0f2a-4a31-bc1f-16f9237d39d6\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T01:28:43.185485913Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T01:30:19.825677573Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0008e25e0\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-rnjkhd\" to finish creating: GET https://rp.j8910848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b86a247-0f2a-4a31-bc1f-16f9237d39d6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b86a247-0f2a-4a31-bc1f-16f9237d39d6\",\n \"name\": \"0b86a247-0f2a-4a31-bc1f-16f9237d39d6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-13T01:28:43.185485913Z\",\n \"endTime\": \"2026-03-13T01:30:19.825677573Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-rnjkhd\\\" to finish creating: GET https://rp.j8910848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b86a247-0f2a-4a31-bc1f-16f9237d39d6\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b86a247-0f2a-4a31-bc1f-16f9237d39d6\\\",\\n \\\"name\\\": \\\"0b86a247-0f2a-4a31-bc1f-16f9237d39d6\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T01:28:43.185485913Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T01:30:19.825677573Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0008e25c0\u003e{\n msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-rnjkhd\\\" to finish creating: GET https://rp.j8910848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b86a247-0f2a-4a31-bc1f-16f9237d39d6\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b86a247-0f2a-4a31-bc1f-16f9237d39d6\\\",\\n \\\"name\\\": \\\"0b86a247-0f2a-4a31-bc1f-16f9237d39d6\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T01:28:43.185485913Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T01:30:19.825677573Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4431/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032254247348736000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4431,"pr_state":"closed","pr_sha":"aa9ab878cd5a35413d9ec28744b440d213397174","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-13T00:35:44Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"202cdb1a05dabe9c52d05402413c797c9d39c69fe45e439d8b9200d95a202406","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4431/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032254247348736000","occurred_at":"2026-03-13T00:35:44Z","failed_at":"e2e","signature_id":"faa88057e089b7c593f84af679cb7d45a1442940a4ff48605cae1ab909bd8b4b","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0009712e0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-clnmmt\" to finish creating: GET https://rp.j8736000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff7319ad-b9ae-48f2-b920-741ee9ae1f66\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff7319ad-b9ae-48f2-b920-741ee9ae1f66\",\n \"name\": \"ff7319ad-b9ae-48f2-b920-741ee9ae1f66\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-13T01:28:45.308865171Z\",\n \"endTime\": \"2026-03-13T01:29:53.779834101Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-294/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-294/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-clnmmt\\\" to finish creating: GET https://rp.j8736000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff7319ad-b9ae-48f2-b920-741ee9ae1f66\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff7319ad-b9ae-48f2-b920-741ee9ae1f66\\\",\\n \\\"name\\\": \\\"ff7319ad-b9ae-48f2-b920-741ee9ae1f66\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T01:28:45.308865171Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T01:29:53.779834101Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-294/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-294/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0009712c0\u003e{\n msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-clnmmt\\\" to finish creating: GET https://rp.j8736000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff7319ad-b9ae-48f2-b920-741ee9ae1f66\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff7319ad-b9ae-48f2-b920-741ee9ae1f66\\\",\\n \\\"name\\\": \\\"ff7319ad-b9ae-48f2-b920-741ee9ae1f66\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T01:28:45.308865171Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T01:29:53.779834101Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-294/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-294/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"err...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0009712e0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-clnmmt\" to finish creating: GET https://rp.j8736000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff7319ad-b9ae-48f2-b920-741ee9ae1f66\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff7319ad-b9ae-48f2-b920-741ee9ae1f66\",\n \"name\": \"ff7319ad-b9ae-48f2-b920-741ee9ae1f66\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-13T01:28:45.308865171Z\",\n \"endTime\": \"2026-03-13T01:29:53.779834101Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-294/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-294/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-clnmmt\\\" to finish creating: GET https://rp.j8736000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff7319ad-b9ae-48f2-b920-741ee9ae1f66\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff7319ad-b9ae-48f2-b920-741ee9ae1f66\\\",\\n \\\"name\\\": \\\"ff7319ad-b9ae-48f2-b920-741ee9ae1f66\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T01:28:45.308865171Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T01:29:53.779834101Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-294/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-294/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0009712c0\u003e{\n msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-clnmmt\\\" to finish creating: GET https://rp.j8736000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff7319ad-b9ae-48f2-b920-741ee9ae1f66\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff7319ad-b9ae-48f2-b920-741ee9ae1f66\\\",\\n \\\"name\\\": \\\"ff7319ad-b9ae-48f2-b920-741ee9ae1f66\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T01:28:45.308865171Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T01:29:53.779834101Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-294/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-294/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"err...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3e98c737ddf5adf1d17baf66717b4a4028c9b16bc52f1934d0b8ab5c6871b406","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4431/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032254247348736000","occurred_at":"2026-03-13T00:35:44Z","failed_at":"e2e","signature_id":"ee40e59d0871edcff5526285af8b0b07e3f84ff4fd3c892f378a234d7dc0fe89","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000f96280\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-rv9psr\" to finish creating: GET https://rp.j8736000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/408d3a2a-e6fc-42e9-8887-1109174c502e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/408d3a2a-e6fc-42e9-8887-1109174c502e\",\n \"name\": \"408d3a2a-e6fc-42e9-8887-1109174c502e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-13T01:28:46.549364074Z\",\n \"endTime\": \"2026-03-13T01:29:53.782924704Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-188/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-188/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-rv9psr\\\" to finish creating: GET https://rp.j8736000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/408d3a2a-e6fc-42e9-8887-1109174c502e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/408d3a2a-e6fc-42e9-8887-1109174c502e\\\",\\n \\\"name\\\": \\\"408d3a2a-e6fc-42e9-8887-1109174c502e\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T01:28:46.549364074Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T01:29:53.782924704Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-188/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-188/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000f96260\u003e{\n msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-rv9psr\\\" to finish creating: GET https://rp.j8736000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/408d3a2a-e6fc-42e9-8887-1109174c502e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/408d3a2a-e6fc-42e9-8887-1109174c502e\\\",\\n \\\"name\\\": \\\"408d3a2a-e6fc-42e9-8887-1109174c502e\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T01:28:46.549364074Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T01:29:53.782924704Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-188/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-188/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n---------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000f96280\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-rv9psr\" to finish creating: GET https://rp.j8736000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/408d3a2a-e6fc-42e9-8887-1109174c502e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/408d3a2a-e6fc-42e9-8887-1109174c502e\",\n \"name\": \"408d3a2a-e6fc-42e9-8887-1109174c502e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-13T01:28:46.549364074Z\",\n \"endTime\": \"2026-03-13T01:29:53.782924704Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-188/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-188/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-rv9psr\\\" to finish creating: GET https://rp.j8736000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/408d3a2a-e6fc-42e9-8887-1109174c502e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/408d3a2a-e6fc-42e9-8887-1109174c502e\\\",\\n \\\"name\\\": \\\"408d3a2a-e6fc-42e9-8887-1109174c502e\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T01:28:46.549364074Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T01:29:53.782924704Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-188/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-188/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000f96260\u003e{\n msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-rv9psr\\\" to finish creating: GET https://rp.j8736000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/408d3a2a-e6fc-42e9-8887-1109174c502e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/408d3a2a-e6fc-42e9-8887-1109174c502e\\\",\\n \\\"name\\\": \\\"408d3a2a-e6fc-42e9-8887-1109174c502e\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T01:28:46.549364074Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T01:29:53.782924704Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-188/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-188/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n---------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"591e0d41498e21b70135cbba4b48fe15cff6e500c66407c195aebfff593c131c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4431/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032254247348736000","occurred_at":"2026-03-13T00:35:44Z","failed_at":"e2e","signature_id":"3b06a8b0231858b9baeb3aa2dd4596ee7d17866c36f5e83504a1635a13fef898","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000874260\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-ql5t9m\" to finish creating: GET https://rp.j8736000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c63d8c71-886e-4886-8a26-caca36c2a4aa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c63d8c71-886e-4886-8a26-caca36c2a4aa\",\n \"name\": \"c63d8c71-886e-4886-8a26-caca36c2a4aa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-13T01:28:45.12883618Z\",\n \"endTime\": \"2026-03-13T01:29:53.774390534Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-ql5t9m\\\" to finish creating: GET https://rp.j8736000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c63d8c71-886e-4886-8a26-caca36c2a4aa\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c63d8c71-886e-4886-8a26-caca36c2a4aa\\\",\\n \\\"name\\\": \\\"c63d8c71-886e-4886-8a26-caca36c2a4aa\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T01:28:45.12883618Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T01:29:53.774390534Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000874240\u003e{\n msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-ql5t9m\\\" to finish creating: GET https://rp.j8736000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c63d8c71-886e-4886-8a26-caca36c2a4aa\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c63d8c71-886e-4886-8a26-caca36c2a4aa\\\",\\n \\\"name\\\": \\\"c63d8c71-886e-4886-8a26-caca36c2a4aa\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T01:28:45.12883618Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T01:29:53.774390534Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-----------------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000874260\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-ql5t9m\" to finish creating: GET https://rp.j8736000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c63d8c71-886e-4886-8a26-caca36c2a4aa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c63d8c71-886e-4886-8a26-caca36c2a4aa\",\n \"name\": \"c63d8c71-886e-4886-8a26-caca36c2a4aa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-13T01:28:45.12883618Z\",\n \"endTime\": \"2026-03-13T01:29:53.774390534Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-ql5t9m\\\" to finish creating: GET https://rp.j8736000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c63d8c71-886e-4886-8a26-caca36c2a4aa\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c63d8c71-886e-4886-8a26-caca36c2a4aa\\\",\\n \\\"name\\\": \\\"c63d8c71-886e-4886-8a26-caca36c2a4aa\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T01:28:45.12883618Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T01:29:53.774390534Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000874240\u003e{\n msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-ql5t9m\\\" to finish creating: GET https://rp.j8736000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c63d8c71-886e-4886-8a26-caca36c2a4aa\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c63d8c71-886e-4886-8a26-caca36c2a4aa\\\",\\n \\\"name\\\": \\\"c63d8c71-886e-4886-8a26-caca36c2a4aa\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T01:28:45.12883618Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T01:29:53.774390534Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-----------------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8d41dbb4afc168272fcc7b1e4e369bcd827a4cf8b958be8970aa59f68bfe1ade","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4431/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032254247348736000","occurred_at":"2026-03-13T00:35:44Z","failed_at":"e2e","signature_id":"dfdb820014955cd92a310b0e59c5e08453113d2a77d6074d241bbd6f7250d33b","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0000e11c0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-2xqtxp\" to finish creating: GET https://rp.j8736000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d365fd38-8477-445b-806e-f8434944b449\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d365fd38-8477-445b-806e-f8434944b449\",\n \"name\": \"d365fd38-8477-445b-806e-f8434944b449\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-13T01:28:45.604723437Z\",\n \"endTime\": \"2026-03-13T01:29:53.794802408Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-296/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-296/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster ea-cluster: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-2xqtxp\\\" to finish creating: GET https://rp.j8736000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d365fd38-8477-445b-806e-f8434944b449\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d365fd38-8477-445b-806e-f8434944b449\\\",\\n \\\"name\\\": \\\"d365fd38-8477-445b-806e-f8434944b449\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T01:28:45.604723437Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T01:29:53.794802408Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-296/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-296/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0000e1180\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-2xqtxp\\\" to finish creating: GET https://rp.j8736000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d365fd38-8477-445b-806e-f8434944b449\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d365fd38-8477-445b-806e-f8434944b449\\\",\\n \\\"name\\\": \\\"d365fd38-8477-445b-806e-f8434944b449\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T01:28:45.604723437Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T01:29:53.794802408Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-296/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-296/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0000e11c0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-2xqtxp\" to finish creating: GET https://rp.j8736000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d365fd38-8477-445b-806e-f8434944b449\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d365fd38-8477-445b-806e-f8434944b449\",\n \"name\": \"d365fd38-8477-445b-806e-f8434944b449\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-13T01:28:45.604723437Z\",\n \"endTime\": \"2026-03-13T01:29:53.794802408Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-296/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-296/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster ea-cluster: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-2xqtxp\\\" to finish creating: GET https://rp.j8736000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d365fd38-8477-445b-806e-f8434944b449\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d365fd38-8477-445b-806e-f8434944b449\\\",\\n \\\"name\\\": \\\"d365fd38-8477-445b-806e-f8434944b449\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T01:28:45.604723437Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T01:29:53.794802408Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-296/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-296/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0000e1180\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-2xqtxp\\\" to finish creating: GET https://rp.j8736000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d365fd38-8477-445b-806e-f8434944b449\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d365fd38-8477-445b-806e-f8434944b449\\\",\\n \\\"name\\\": \\\"d365fd38-8477-445b-806e-f8434944b449\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-13T01:28:45.604723437Z\\\",\\n \\\"endTime\\\": \\\"2026-03-13T01:29:53.794802408Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-296/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-296/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032254229569081344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4431,"pr_state":"closed","pr_sha":"6e42c179e772e0c68437318b6667b962c229876e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-13T00:35:40Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ff286c8403b5c7136fe55f5cc661bc2aef9dd01fe1ed2be09a1c8eb4bc043815","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032254229569081344","occurred_at":"2026-03-13T00:35:40Z","failed_at":"provision","signature_id":"c555358b934034266edbb8c57102dc3e1e5c798ea0efd005aba670949dd66cdc","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-velero-image","test_suite":"step graph","failure_text":"time=2026-03-13T01:34:04.086Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image description=\"Step mirror-velero-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e velero.server.registry \u003cnil\u003e}:{\u003cnil\u003e velero.server.repository \u003cnil\u003e}@{\u003cnil\u003e velero.server.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-13T01:34:14.742Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org\\nFetch pull secret for source registry registry.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nFailed to log in to ACR arohcpsvcdev: ERROR: Error loading command module 'acs': deadlock detected by _ModuleLock('azure.core.pipeline.transport') at 140122189728464\\nERROR: Error loading command module 'batch': cannot import name 'PipelineClient' from 'azure.core' (/usr/lib64/az/lib/python3.12/site-packages/azure/core/__init__.py)\\nERROR: Error loading command module 'acr': cannot import name 'SpanKind' from partially initialized module 'azure.core.tracing' (most likely due to a circular import) (/usr/lib64/az/lib/python3.12/site-packages/azure/core/tracing/__init__.py)\\nERROR: Error loading command module 'appservice': cannot import name 'PipelineClient' from 'azure.core' (/usr/lib64/az/lib/python3.12/site-packages/azure/core/__init__.py)\\nERROR: 'acr' is misspelled or not recognized by the system.\\n\\nExamples from AI knowledge base:\\nhttps://aka.ms/cli_ref\\nRead more about the command in reference docs\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"41cea61b940716547ec2265592f122e93352921768862338021f0fc077c6bca6","failure_pattern":"ERROR: 'acr' is misspelled or not recognized by the system.","search_query":"ERROR: 'acr' is misspelled or not recognized by the system."}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["41cea61b940716547ec2265592f122e93352921768862338021f0fc077c6bca6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032253527601975296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4399,"pr_state":"closed","pr_sha":"8a1ae2ac7a78b624556d264008ca37e8c812709f","final_merged_sha":"8a1ae2ac7a78b624556d264008ca37e8c812709f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-13T00:32:53Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4409/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032252938402926592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4409,"pr_state":"closed","pr_sha":"8814a92f744ea05a873a749208b6214644fb124e","final_merged_sha":"61fd73675b4e600ec10d87e4a1ef0ac193215b57","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-13T00:30:32Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"6c21f0d7bd7ed79b5f43cc4a214e7b0fb658fc7353e8e3960eec81a2916af197","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4409/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032252938402926592","occurred_at":"2026-03-13T00:30:32Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032252834484850688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4399,"pr_state":"closed","pr_sha":"8a1ae2ac7a78b624556d264008ca37e8c812709f","final_merged_sha":"8a1ae2ac7a78b624556d264008ca37e8c812709f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-13T00:30:07Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f618a49a6e48ef263202ffff66070d1ed1beb3e9f7a66317215a3f0550ef14bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032252834484850688","occurred_at":"2026-03-13T00:30:07Z","failed_at":"e2e","signature_id":"1fc065e1a53fa9fa868ffe68fc5a1edf5e5a73913eb3a54fef44fca288384296","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:314]: Told to stop trying after 20.278s.\nPod failed; see logs for details\nAt one point, however, the function did return successfully.\nYet, Eventually failed because the matcher was not satisfied:\nExpected\n \u003cv1.PodPhase\u003e: Running\nto equal\n \u003cv1.PodPhase\u003e: Succeeded\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:314]: Told to stop trying after 20.278s.\nPod failed; see logs for details\nAt one point, however, the function did return successfully.\nYet, Eventually failed because the matcher was not satisfied:\nExpected\n \u003cv1.PodPhase\u003e: Running\nto equal\n \u003cv1.PodPhase\u003e: Succeeded","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2c8d18cdb2b7dd33e8c21a8b6a9a4ce7f6faa5fa98e5020989e6c4479385b197","failure_pattern":"Yet, Eventually failed because the matcher was not satisfied:","search_query":"Yet, Eventually failed because the matcher was not satisfied:"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2c8d18cdb2b7dd33e8c21a8b6a9a4ce7f6faa5fa98e5020989e6c4479385b197"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032252165443031040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4390,"pr_state":"closed","pr_sha":"2cc3d4711454f1e97a3dbc84a78d911569b181bc","final_merged_sha":"2cc3d4711454f1e97a3dbc84a78d911569b181bc","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-13T00:27:28Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"ea1b337393bf72d29e16521e9162e9ad5bb60545b4c4ca29759184c7b73a783f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032252165443031040","occurred_at":"2026-03-13T00:27:28Z","failed_at":"e2e","signature_id":"9f2d8ac4d8e315a897508ff607f8dee91a16629783b0a3f15365667c52283f98","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:314]: Told to stop trying after 20.276s.\nPod failed; see logs for details\nAt one point, however, the function did return successfully.\nYet, Eventually failed because the matcher was not satisfied:\nExpected\n \u003cv1.PodPhase\u003e: Running\nto equal\n \u003cv1.PodPhase\u003e: Succeeded\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:314]: Told to stop trying after 20.276s.\nPod failed; see logs for details\nAt one point, however, the function did return successfully.\nYet, Eventually failed because the matcher was not satisfied:\nExpected\n \u003cv1.PodPhase\u003e: Running\nto equal\n \u003cv1.PodPhase\u003e: Succeeded","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2c8d18cdb2b7dd33e8c21a8b6a9a4ce7f6faa5fa98e5020989e6c4479385b197","failure_pattern":"Yet, Eventually failed because the matcher was not satisfied:","search_query":"Yet, Eventually failed because the matcher was not satisfied:"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2c8d18cdb2b7dd33e8c21a8b6a9a4ce7f6faa5fa98e5020989e6c4479385b197"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]}]}