{"meta":{"date":"2026-03-16","timezone":"UTC","generated_at":"2026-10-31T23:15:12Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":199,"failed_runs":185,"runs_with_occurrences":185,"runs_with_failure_pattern_matches":78,"runs_with_unmatched_occurrences":107,"runs_without_occurrence_rows":14,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033694230072791040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4433,"pr_state":"closed","pr_sha":"599dd2c53ef9a79607254436be4a2ed8e0671e8c","final_merged_sha":"599dd2c53ef9a79607254436be4a2ed8e0671e8c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-16T23:57:43Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4461/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033687878516609024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4461,"pr_state":"closed","pr_sha":"421438d23c10f9e39e468effb1aad24a3674531d","final_merged_sha":"9baa329552156f993c1fc5b382b78423cd754fef","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T23:32:28Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"5fed5e53c362c2c7e2db30b503386b72f869bab7bc76ecd76f85293eeef954e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4461/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033687878516609024","occurred_at":"2026-03-16T23:32:28Z","failed_at":"provision","signature_id":"543c429be64f4ea15622f0c16cf8a4b86c604206ca94730d88c051d303ca55ea","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/acrpull","test_suite":"step graph","failure_text":"time=2026-03-17T00:03:10.626Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=acrpull description=\"Step acrpull\\n Kind: Helm\\n\"\ntime=2026-03-17T00:03:11.206Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=acrpull values=\"map[image:mcr.microsoft.com/aks/msi-acrpull@sha256:042511defb0842f02593e48e68d83a20a1af4265db4e23fca484356e34a6c883 namespace:acrpull replicas:2 ttlRotationFraction:0.5]\"\ntime=2026-03-17T00:03:11.206Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=acrpull\ntime=2026-03-17T00:03:11.515Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=acrpull namespace=acrpull\ntime=2026-03-17T00:03:11.808Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=acrpull dryRun=false\ntime=2026-03-17T00:03:11.808Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=acrpull\ntime=2026-03-17T00:03:11.923Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=acrpull\ntime=2026-03-17T00:08:19.495Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=acrpull err=\"resource not ready, name: acrpull, kind: Deployment, status: Unknown\\ncontext deadline exceeded\"\ntime=2026-03-17T00:08:19.495Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=acrpull\ntime=2026-03-17T00:08:19.495Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=acrpull\ntime=2026-03-17T00:08:23.270Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=acrpull release=acrpull namespace=acrpull status=failed description=\"Release \\\"acrpull\\\" failed: resource not ready, name: acrpull, kind: Deployment, status: Unknown\\ncontext deadline exceeded\" values=\"map[image:mcr.microsoft.com/aks/msi-acrpull@sha256:042511defb0842f02593e48e68d83a20a1af4265db4e23fca484356e34a6c883 namespace:acrpull replicas:2 ttlRotationFraction:0.5]\"\ntime=2026-03-17T00:08:23.272Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=acrpull url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/6yUzW7TQBCA736KkU+JZEdJK/ET1ENJOaGiqiAORRzWuxMzzXrG7I5bRcrDo8QNXVoS0oijZ7/95metyTwqBIzSBYsRzsAZNWoqjwOpbtHqR2I3jRqI6wL60CfT4O8Q3iFvIrE1dhsefssAAPJZF1Wa6wf9Bc6JSUk4LyA3NrSd9xWxI67j6OF71JANEmWuIyvNGsyLl8uaSOU/hdfiMblcWmEN4j2G0qNxGEr0aJ+kSC+/79Md5iirZ/TWdYGtl2WDrH8cPuOuxF0Kk0rYz301npxR4vrcNRQjCV+JJ7tMCk4KVVkglwF/dhi1bDdkWu2R2og2oJZNp2bT/38UhzuyWBprpeODMsx8FxXD7hffAe9/4Ze0csDMj/H8fcjHmfZONVV+7tHzntz/L86E51RfmnZztq2glxLGdJx2g6aO7Pu7bNFVGBgV44f1qonZCu5/YEBQajCqaVqoUO8RGQbOKK7Dg/xkfPKqHJ+Wk9dfxuPp+HQ6mdzkQxiNYCfzZjp5e5MPh9kKboUYFsTujJgxJAtSGB73YroQn27CbAVtkPXpY6EFtOJ6+DBJAV7qAhqM0dRrpQSHAapl0rzDaH8FAAD//wniDsXGBQAA\"\ntime=2026-03-17T00:08:23.274Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=acrpull\ntime=2026-03-17T00:08:23.274Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=acrpull err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: acrpull, kind: Deployment, status: Unknown\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e691b57c8f52b6f4d72b181f4fd08c35a630866258dbcb5781e97fa869159de1","failure_pattern":"resource not ready, name: acrpull, kind: Deployment, status: Unknown context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: acrpull, kind: Deployment, status: Unknown\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e691b57c8f52b6f4d72b181f4fd08c35a630866258dbcb5781e97fa869159de1"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4486/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033686064194916352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4486,"pr_state":"closed","pr_sha":"2c15e1139e1bc7f2fe0b42b8a415009708f04066","final_merged_sha":"2c15e1139e1bc7f2fe0b42b8a415009708f04066","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T23:25:16Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"1ffd145f0fe51e4fcd22ffad50f58545e1894f80b51f6110ff1ee1d791bad4b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4486/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033686064194916352","occurred_at":"2026-03-16T23:25:16Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033680150008172544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4482,"pr_state":"closed","pr_sha":"0b90840021e1fdc816441a8107f7b8aa42d1b0ed","final_merged_sha":"04f43f02c3600758a856fe0f5464f9937b913cdd","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T23:01:46Z"},"failed_at":["e2e"],"failed_test_count":7,"occurrences":[{"row_id":"39524ae07c06f4b5715f25626e5cbbe0c9c3e0ec7dfc0c2b4ee655e9d3ff0409","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033680150008172544","occurred_at":"2026-03-16T23:01:46Z","failed_at":"e2e","signature_id":"85a778ca549b53e44fccbe081d4c27f1114c85711390e3a267823f7736fdf3a6","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00141f530\u003e: \n failed to create HCP cluster cluster-zstream-4-20-g88rq7, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-g88rq7 in resource group rg-zstream-upgrade-4-20-qg77d2, error: failed waiting for cluster=\"cluster-zstream-4-20-g88rq7\" in resourcegroup=\"rg-zstream-upgrade-4-20-qg77d2\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-g88rq7 in resource group rg-zstream-upgrade-4-20-qg77d2, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster cluster-zstream-4-20-g88rq7, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-g88rq7 in resource group rg-zstream-upgrade-4-20-qg77d2, error: failed waiting for cluster=\\\"cluster-zstream-4-20-g88rq7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-qg77d2\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-g88rq7 in resource group rg-zstream-upgrade-4-20-qg77d2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013b6420\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-g88rq7 in resource group rg-zstream-upgrade-4-20-qg77d2\",\n },\n \u003c*fmt.wrapErrors | 0xc00141f500\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-g88rq7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-qg77d2\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-g88rq7 in resource group rg-zstream-upgrade-4-20-qg77d2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013b6420\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-g88rq7 in resource group rg-zstream-upgrade-4-20-qg77d2\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00141f530\u003e: \n failed to create HCP cluster cluster-zstream-4-20-g88rq7, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-g88rq7 in resource group rg-zstream-upgrade-4-20-qg77d2, error: failed waiting for cluster=\"cluster-zstream-4-20-g88rq7\" in resourcegroup=\"rg-zstream-upgrade-4-20-qg77d2\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-g88rq7 in resource group rg-zstream-upgrade-4-20-qg77d2, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster cluster-zstream-4-20-g88rq7, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-g88rq7 in resource group rg-zstream-upgrade-4-20-qg77d2, error: failed waiting for cluster=\\\"cluster-zstream-4-20-g88rq7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-qg77d2\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-g88rq7 in resource group rg-zstream-upgrade-4-20-qg77d2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013b6420\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-g88rq7 in resource group rg-zstream-upgrade-4-20-qg77d2\",\n },\n \u003c*fmt.wrapErrors | 0xc00141f500\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-g88rq7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-qg77d2\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-g88rq7 in resource group rg-zstream-upgrade-4-20-qg77d2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013b6420\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-g88rq7 in resource group rg-zstream-upgrade-4-20-qg77d2\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-g88rq7 in resource group rg-zstream-upgrade-4-20-qg77d2, error: failed waiting for cluster=\"cluster-zstream-4-20-g88rq7\" in resourcegroup=\"rg-zstream-upgrade-4-20-qg77d2\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-g88rq7 in resource group rg-zstream-upgrade-4-20-qg77d2, error: context deadline exceeded"}},{"row_id":"61c7dd95f761ee6a256d01f9f13392b2095406b49dee3840c6ed0087f3c728eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033680150008172544","occurred_at":"2026-03-16T23:01:46Z","failed_at":"e2e","signature_id":"36cc2b79340b35a1380a06aa6e056eebcba0f368dc835beed0452f7afe106376","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0010ec0e0\u003e: \n failed to create HCP cluster cluster-zstream-4-19-jlc5tn: failed starting cluster creation \"cluster-zstream-4-19-jlc5tn\" in resourcegroup=\"rg-zstream-upgrade-4-19-rdkr4c\": PUT https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-rdkr4c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-jlc5tn\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-jlc5tn: failed starting cluster creation \\\"cluster-zstream-4-19-jlc5tn\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-rdkr4c\\\": PUT https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-rdkr4c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-jlc5tn\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0010ec0c0\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-jlc5tn\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-rdkr4c\\\": PUT https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-rdkr4c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-jlc5tn\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001310300\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"956b09c4-e7a8-41ec-a551-201abc827d34\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 23:51:33 GMT\",\n ],\n \"Content-Length\": [\"107\"],\n \"X-Envoy-Upstream-Service-Time\": [\"9547\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0010ec000\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j8172544.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-rdkr4c/pr...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0010ec0e0\u003e: \n failed to create HCP cluster cluster-zstream-4-19-jlc5tn: failed starting cluster creation \"cluster-zstream-4-19-jlc5tn\" in resourcegroup=\"rg-zstream-upgrade-4-19-rdkr4c\": PUT https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-rdkr4c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-jlc5tn\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-jlc5tn: failed starting cluster creation \\\"cluster-zstream-4-19-jlc5tn\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-rdkr4c\\\": PUT https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-rdkr4c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-jlc5tn\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0010ec0c0\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-jlc5tn\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-rdkr4c\\\": PUT https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-rdkr4c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-jlc5tn\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001310300\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"956b09c4-e7a8-41ec-a551-201abc827d34\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 23:51:33 GMT\",\n ],\n \"Content-Length\": [\"107\"],\n \"X-Envoy-Upstream-Service-Time\": [\"9547\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0010ec000\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j8172544.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-rdkr4c/pr...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"80a9444995c4cee4ec9f6a73b33345d7381ff580b07ad6ab87613f3d914cac9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033680150008172544","occurred_at":"2026-03-16T23:01:46Z","failed_at":"e2e","signature_id":"f5abe33b9270faf2bd0b533b6aed2d4d48e7fc60a49ac959ac85a871e52763e6","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:400]: Unexpected error:\n \u003c*exported.ResponseError | 0xc0014fa090\u003e: \n GET https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-tm62sq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"107\"],\n \"X-Envoy-Upstream-Service-Time\": [\"3208\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"9ea98782-3e1d-4675-95fc-91b3b30cbc83\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 23:51:50 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00116ad40\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"GET\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j8172544.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-tm62sq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\",\n RawPath: \"\",\n OmitHost: false,\n ForceQuery: false,\n RawQuery: \"api-version=2024-06-10-preview\",\n Fragment: \"\",\n RawFragment: \"\",\n },\n Proto: \"HTTP/1.1\",\n ProtoMajor: 1,\n ProtoMinor: 1,\n Header: {\n \"X-Ms-Identity-Url\": [\n \"https://dummyhost.identity.azure.net\",\n ],\n \"Accept\": [\"application/json\"],\n \"User-Agent\": [\n \"azsdk-go-armredhatopenshifthcp/v0.1.0 (go1.25.7; linux)\",\n ],\n \"Authorization\": [\"redacted\"],\n \"X-Ms-Arm-Resource-System-Data\": [\n \"{\\\"createdBy\\\": \\\"e2e-test\\\", \\\"createdByType\\\": \\\"Application\\\", \\\"createdAt\\\": \\\"2026-03-16T23:51:27Z\\\"}\",\n ],\n },\n Body: nil,\n GetBody: nil,\n ContentLength: 0,\n TransferEncoding: nil,\n Close: false,\n Host: \"rp.j8172544.hcpsvc.osadev.cloud\",\n Form: nil,\n PostForm: nil,\n MultipartForm: nil,\n Trailer: nil,\n RemoteAddr: \"\",\n RequestURI: \"\",\n TLS: nil,\n Cancel: nil,\n Response: nil,\n Pattern: \"\",\n ctx: \u003csuppressed context\u003e,\n pat: nil,\n matches: nil,\n otherValues: nil,\n },\n TLS: {\n Version: 772,\n HandshakeComplete: true,\n DidResume: false,\n CipherSuite: 4865,\n CurveID: 29,\n NegotiatedProtocol: \"h2\",\n NegotiatedProtocolIsMutual: true,\n ServerName: \"rp.j8172544.hcpsvc.osadev.cl...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:400]: Unexpected error:\n \u003c*exported.ResponseError | 0xc0014fa090\u003e: \n GET https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-tm62sq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"107\"],\n \"X-Envoy-Upstream-Service-Time\": [\"3208\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"9ea98782-3e1d-4675-95fc-91b3b30cbc83\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 23:51:50 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00116ad40\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"GET\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j8172544.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-tm62sq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\",\n RawPath: \"\",\n OmitHost: false,\n ForceQuery: false,\n RawQuery: \"api-version=2024-06-10-preview\",\n Fragment: \"\",\n RawFragment: \"\",\n },\n Proto: \"HTTP/1.1\",\n ProtoMajor: 1,\n ProtoMinor: 1,\n Header: {\n \"X-Ms-Identity-Url\": [\n \"https://dummyhost.identity.azure.net\",\n ],\n \"Accept\": [\"application/json\"],\n \"User-Agent\": [\n \"azsdk-go-armredhatopenshifthcp/v0.1.0 (go1.25.7; linux)\",\n ],\n \"Authorization\": [\"redacted\"],\n \"X-Ms-Arm-Resource-System-Data\": [\n \"{\\\"createdBy\\\": \\\"e2e-test\\\", \\\"createdByType\\\": \\\"Application\\\", \\\"createdAt\\\": \\\"2026-03-16T23:51:27Z\\\"}\",\n ],\n },\n Body: nil,\n GetBody: nil,\n ContentLength: 0,\n TransferEncoding: nil,\n Close: false,\n Host: \"rp.j8172544.hcpsvc.osadev.cloud\",\n Form: nil,\n PostForm: nil,\n MultipartForm: nil,\n Trailer: nil,\n RemoteAddr: \"\",\n RequestURI: \"\",\n TLS: nil,\n Cancel: nil,\n Response: nil,\n Pattern: \"\",\n ctx: \u003csuppressed context\u003e,\n pat: nil,\n matches: nil,\n otherValues: nil,\n },\n TLS: {\n Version: 772,\n HandshakeComplete: true,\n DidResume: false,\n CipherSuite: 4865,\n CurveID: 29,\n NegotiatedProtocol: \"h2\",\n NegotiatedProtocolIsMutual: true,\n ServerName: \"rp.j8172544.hcpsvc.osadev.cl...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8fc5fabd0109eba64ee47c81eb2ecaf53820b5e8e9cea624fadc6e7b9cdc79bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033680150008172544","occurred_at":"2026-03-16T23:01:46Z","failed_at":"e2e","signature_id":"1c8f3f28ba33b0a272c332a5c5d0606aefaae40036315a5ec51de64173ec1db6","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0006678a0\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed starting cluster creation \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-rrmlfs\": PUT https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-rrmlfs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster: failed starting cluster creation \\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-rrmlfs\\\": PUT https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-rrmlfs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0006677e0\u003e{\n msg: \"failed starting cluster creation \\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-rrmlfs\\\": PUT https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-rrmlfs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000aa3800\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Envoy-Upstream-Service-Time\": [\"3945\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"415a824d-9904-4ac6-84ed-d1fb2146174a\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 23:51:54 GMT\",\n ],\n \"Content-Length\": [\"107\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000667640\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j8172544.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-rrmlfs/providers/Mi...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0006678a0\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed starting cluster creation \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-rrmlfs\": PUT https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-rrmlfs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster: failed starting cluster creation \\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-rrmlfs\\\": PUT https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-rrmlfs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0006677e0\u003e{\n msg: \"failed starting cluster creation \\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-rrmlfs\\\": PUT https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-rrmlfs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000aa3800\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Envoy-Upstream-Service-Time\": [\"3945\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"415a824d-9904-4ac6-84ed-d1fb2146174a\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 23:51:54 GMT\",\n ],\n \"Content-Length\": [\"107\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000667640\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j8172544.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-rrmlfs/providers/Mi...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a65df1a0a83c7abf0a4410ed0faff4321833477ffd62b3454a486d6aa85cc314","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033680150008172544","occurred_at":"2026-03-16T23:01:46Z","failed_at":"e2e","signature_id":"80587e6475d085870322838edef85a8048444d0350c81feac0a7f5c45a0bdb54","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243]: Unexpected error:\n \u003c*errors.joinError | 0xc00179b4a0\u003e: \n failed to cleanup resource group: failed listing hcp clusters in resourcegroup=\"external-auth-cluster-wdzps7\": GET https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-wdzps7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n errs: [\n \u003c*fmt.wrapError | 0xc0012c71a0\u003e{\n msg: \"failed to cleanup resource group: failed listing hcp clusters in resourcegroup=\\\"external-auth-cluster-wdzps7\\\": GET https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-wdzps7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0012c7180\u003e{\n msg: \"failed listing hcp clusters in resourcegroup=\\\"external-auth-cluster-wdzps7\\\": GET https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-wdzps7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0010f29c0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"78f9e056-a4a6-4a4c-9502-b92baf96f96a\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 23:50:41 GMT\",\n ],\n \"Content-Length\": [\"107\"],\n \"X-Envoy-Upstream-Service-Time\": [\"2880\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0012c70c0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"GET\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243]: Unexpected error:\n \u003c*errors.joinError | 0xc00179b4a0\u003e: \n failed to cleanup resource group: failed listing hcp clusters in resourcegroup=\"external-auth-cluster-wdzps7\": GET https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-wdzps7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n errs: [\n \u003c*fmt.wrapError | 0xc0012c71a0\u003e{\n msg: \"failed to cleanup resource group: failed listing hcp clusters in resourcegroup=\\\"external-auth-cluster-wdzps7\\\": GET https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-wdzps7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0012c7180\u003e{\n msg: \"failed listing hcp clusters in resourcegroup=\\\"external-auth-cluster-wdzps7\\\": GET https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-wdzps7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0010f29c0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"78f9e056-a4a6-4a4c-9502-b92baf96f96a\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 23:50:41 GMT\",\n ],\n \"Content-Length\": [\"107\"],\n \"X-Envoy-Upstream-Service-Time\": [\"2880\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0012c70c0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"GET\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c11a105422e57656a5c9fd9dcf3e28fb21d1e8d7178db11e21a6160a745ecd7e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033680150008172544","occurred_at":"2026-03-16T23:01:46Z","failed_at":"e2e","signature_id":"85af157c7b24c4d4daf3342b6847b1ce13cd6ed04d600d0f62a69fd7dd0d78a5","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:247]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0011ba3a0\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-7vk4vx\" in resourcegroup=\"admin-credential-lifecycle-test-w9rq7p\" to finish getting creds: GET https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationResults/1f808343-553b-459a-bd27-b8d61314ce16\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-7vk4vx\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-w9rq7p\\\" to finish getting creds: GET https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationResults/1f808343-553b-459a-bd27-b8d61314ce16\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0007ef7d0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"107\"],\n \"X-Envoy-Upstream-Service-Time\": [\"2901\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"311609f5-9d4d-4266-8e59-b7148893c123\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 23:50:27 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0011ba2c0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"GET\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j8172544.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationResults/1f808343-553b-459a-bd27-b8d61314ce16\",\n RawPath: \"\",\n OmitHost: false,\n ForceQuery: false,\n RawQuery: \"api-version=2024-06-10-preview\",\n Fragment: \"\",\n RawFragment: \"\",\n },\n Proto: \"HTTP/1.1\",\n ProtoMajor: 1,\n ProtoMinor: 1,\n Header: {\n \"User-Agent\": [\n \"azsdk-go-armredhatopenshifthcp/v0.1.0 (go1.25.7; linux)\",\n ],\n \"X-Ms-Arm-Resource-System-Data\": [\n \"{\\\"createdBy\\\": \\\"e2e-test\\\", \\\"createdByType\\\": \\\"Application\\\", \\\"createdAt\\\": \\\"2026-03-16T23:50:07Z\\\"}\",\n ],\n \"X-Ms-Identity-Url\": [\n \"https://dummyhost.identity.azure.net\",\n ],\n \"Authorization\": [...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:247]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0011ba3a0\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-7vk4vx\" in resourcegroup=\"admin-credential-lifecycle-test-w9rq7p\" to finish getting creds: GET https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationResults/1f808343-553b-459a-bd27-b8d61314ce16\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-7vk4vx\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-w9rq7p\\\" to finish getting creds: GET https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationResults/1f808343-553b-459a-bd27-b8d61314ce16\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0007ef7d0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"107\"],\n \"X-Envoy-Upstream-Service-Time\": [\"2901\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"311609f5-9d4d-4266-8e59-b7148893c123\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 23:50:27 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0011ba2c0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"GET\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j8172544.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationResults/1f808343-553b-459a-bd27-b8d61314ce16\",\n RawPath: \"\",\n OmitHost: false,\n ForceQuery: false,\n RawQuery: \"api-version=2024-06-10-preview\",\n Fragment: \"\",\n RawFragment: \"\",\n },\n Proto: \"HTTP/1.1\",\n ProtoMajor: 1,\n ProtoMinor: 1,\n Header: {\n \"User-Agent\": [\n \"azsdk-go-armredhatopenshifthcp/v0.1.0 (go1.25.7; linux)\",\n ],\n \"X-Ms-Arm-Resource-System-Data\": [\n \"{\\\"createdBy\\\": \\\"e2e-test\\\", \\\"createdByType\\\": \\\"Application\\\", \\\"createdAt\\\": \\\"2026-03-16T23:50:07Z\\\"}\",\n ],\n \"X-Ms-Identity-Url\": [\n \"https://dummyhost.identity.azure.net\",\n ],\n \"Authorization\": [...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fb90fbab74c1f7df70760da61c757e081389523c694c54e8a391b6219ed6ac21","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033680150008172544","occurred_at":"2026-03-16T23:01:46Z","failed_at":"e2e","signature_id":"63b428ca26d54da1536a645953aa5ae83499f7099b0a6e6b192f44899b4b884e","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000730520\u003e: \n failed to create HCP cluster cluster-zstream-4-21-x4bct2: failed starting cluster creation \"cluster-zstream-4-21-x4bct2\" in resourcegroup=\"rg-zstream-upgrade-4-21-zbvp7q\": PUT https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-zbvp7q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-x4bct2\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-21-x4bct2: failed starting cluster creation \\\"cluster-zstream-4-21-x4bct2\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-zbvp7q\\\": PUT https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-zbvp7q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-x4bct2\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000730500\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-21-x4bct2\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-zbvp7q\\\": PUT https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-zbvp7q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-x4bct2\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000988180\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Envoy-Upstream-Service-Time\": [\"3181\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"5408bd3e-ffe3-4a8e-a5cb-61938c9b2d00\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 23:51:51 GMT\",\n ],\n \"Content-Length\": [\"107\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000730160\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j8172544.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-zbvp7q/pr...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000730520\u003e: \n failed to create HCP cluster cluster-zstream-4-21-x4bct2: failed starting cluster creation \"cluster-zstream-4-21-x4bct2\" in resourcegroup=\"rg-zstream-upgrade-4-21-zbvp7q\": PUT https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-zbvp7q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-x4bct2\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-21-x4bct2: failed starting cluster creation \\\"cluster-zstream-4-21-x4bct2\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-zbvp7q\\\": PUT https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-zbvp7q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-x4bct2\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000730500\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-21-x4bct2\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-zbvp7q\\\": PUT https://rp.j8172544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-zbvp7q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-x4bct2\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000988180\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Envoy-Upstream-Service-Time\": [\"3181\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"5408bd3e-ffe3-4a8e-a5cb-61938c9b2d00\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 23:51:51 GMT\",\n ],\n \"Content-Length\": [\"107\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000730160\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j8172544.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-zbvp7q/pr...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":7,"failure_pattern_ids":["223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6","f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":7,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033669775233650688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4482,"pr_state":"closed","pr_sha":"0b90840021e1fdc816441a8107f7b8aa42d1b0ed","final_merged_sha":"04f43f02c3600758a856fe0f5464f9937b913cdd","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T22:20:32Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"18aaaa16737e6575dd68c760bdcc4a96e22713fc7d25970135923b344b39dc79","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033669775233650688","occurred_at":"2026-03-16T22:20:32Z","failed_at":"provision","signature_id":"14e1139051b92a0259280ab4bba3ccc679edd3541b36e2204a8d4ec84f155427","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/prometheus","test_suite":"step graph","failure_text":"time=2026-03-16T22:48:23.283Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-03-16T22:48:23.954Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:d7e8257f8d8bce64b6df55f81fba92011a6a77269b3350f8b997b152af348dba]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:4d4a8749-1fac-4d4f-916f-0e98fd14766f]] prometheusSpec:map[externalLabels:map[cluster:prow-j3650688-svc] hcpRemoteWriteUrl:NONE image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:dca4b35f73046dc4803c0160c2e6be5a1736a716b26ef35590864d36dfdf950a] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j3650688-svc-ezhn.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-557258f7768e454a9792a5c6bd539df9/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-16T22:48:23.955Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-16T22:48:24.319Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus namespace=prometheus\ntime=2026-03-16T22:48:24.730Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus dryRun=false\ntime=2026-03-16T22:48:24.730Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-16T22:48:26.065Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus numVersions=0\ntime=2026-03-16T22:48:26.177Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"\\\"arohcp-monitor\\\" has no deployed releases\"\ntime=2026-03-16T22:48:26.177Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-16T22:48:26.177Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-16T22:48:26.288Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"failed to get status for release arohcp-monitor: release: not found\"\ntime=2026-03-16T22:48:26.288Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-16T22:48:26.289Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: \\\"arohcp-monitor\\\" has no deployed releases\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3f7e199865c5445ff3484bb28bdeda70d99439e135094e3f2bd18d66ac18f0c1","failure_pattern":"\"arohcp-monitor\" has no deployed releases","search_query":"error running Helm release deployment Step, failed to deploy helm release: \\\"arohcp-monitor\\\" has no deployed releases"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["3f7e199865c5445ff3484bb28bdeda70d99439e135094e3f2bd18d66ac18f0c1"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033665202905419776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4454,"pr_state":"closed","pr_sha":"38f866ad42f9613c0978385116cd5b0c6df796c1","final_merged_sha":"38f866ad42f9613c0978385116cd5b0c6df796c1","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T22:02:22Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d43846b0a3facf3243a03376b94d3664c65b48381ed0f6702cd65ae7af003822","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033665202905419776","occurred_at":"2026-03-16T22:02:22Z","failed_at":"provision","signature_id":"5564627504dc0ffa8b2a1cfb61f4b200a8883ded65fa2c1a4914a1dd7bd7f88c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-config","test_suite":"step graph","failure_text":"time=2026-03-16T22:45:32.131Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config description=\"Step deploy-mce-config\\n Kind: Helm\\n\"\ntime=2026-03-16T22:45:32.681Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f global:map[registryOverride:arohcpsvcdev.azurecr.io/acm-d-cache] localCluster:map[addonConfig:map[applicationManagerEnabled:false certPolicyControllerEnabled:false policyControllerEnabled:true searchCollectorEnabled:false] kubeApiUrl:https://kubernetes.default.svc] policy:map[cluster-lifecycle:map[hubconfig:map[replicaCount:2]] grc:map[hubconfig:map[replicaCount:2]]]]\"\ntime=2026-03-16T22:45:32.681Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-16T22:45:32.965Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config namespace=multicluster-engine\ntime=2026-03-16T22:45:33.075Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config dryRun=false\ntime=2026-03-16T22:45:33.075Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-16T22:45:33.182Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-16T22:47:08.180Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"failed post-install: Unauthorized\"\ntime=2026-03-16T22:47:08.180Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-16T22:47:08.180Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-16T22:47:18.981Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"failed to get status for release mce-config: kubernetes cluster unreachable: the server has asked for the client to provide credentials\"\ntime=2026-03-16T22:47:18.981Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-16T22:47:18.981Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"error running Helm release deployment Step, failed to deploy helm release: failed post-install: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82e6580bc32e56aa7378422fd2960d02a2eebbb9fbfbc08994384c40137bbab6","failure_pattern":"failed post-install: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed post-install: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["82e6580bc32e56aa7378422fd2960d02a2eebbb9fbfbc08994384c40137bbab6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4466/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033664276417220608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4466,"pr_state":"closed","pr_sha":"8cb4bd5f1213fd23ab6fcccc158048912ff9661e","final_merged_sha":"8cb4bd5f1213fd23ab6fcccc158048912ff9661e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-16T21:58:41Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4486/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033663933222490112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4486,"pr_state":"closed","pr_sha":"454b8c0c5651e72500a7711db75ae8532746788a","final_merged_sha":"2c15e1139e1bc7f2fe0b42b8a415009708f04066","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T21:57:18Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"afb6232563c94039cfc5b977621a2647748c9a53e06237843f55880d7a642af4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4486/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033663933222490112","occurred_at":"2026-03-16T21:57:18Z","failed_at":"provision","signature_id":"5564627504dc0ffa8b2a1cfb61f4b200a8883ded65fa2c1a4914a1dd7bd7f88c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-config","test_suite":"step graph","failure_text":"time=2026-03-16T22:43:56.374Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config description=\"Step deploy-mce-config\\n Kind: Helm\\n\"\ntime=2026-03-16T22:43:56.974Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f global:map[registryOverride:arohcpsvcdev.azurecr.io/acm-d-cache] localCluster:map[addonConfig:map[applicationManagerEnabled:false certPolicyControllerEnabled:false policyControllerEnabled:true searchCollectorEnabled:false] kubeApiUrl:https://kubernetes.default.svc] policy:map[cluster-lifecycle:map[hubconfig:map[replicaCount:2]] grc:map[hubconfig:map[replicaCount:2]]]]\"\ntime=2026-03-16T22:43:56.974Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-16T22:43:57.031Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config namespace=multicluster-engine\ntime=2026-03-16T22:43:57.145Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config dryRun=false\ntime=2026-03-16T22:43:57.145Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-16T22:43:57.251Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-16T22:45:27.656Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"failed post-install: an error on the server (\\\"Internal Server Error: \\\\\\\"/apis/batch/v1/namespaces/multicluster-engine/jobs/finalize-mce-config\\\\\\\": Post \\\\\\\"https://guard.69b883de32e3a9000188c8bc.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \\\\\\\"crypto/rsa: verification error\\\\\\\" while trying to verify candidate authority certificate \\\\\\\"ca\\\\\\\")\\\") has prevented the request from succeeding (delete jobs.batch finalize-mce-config)\"\ntime=2026-03-16T22:45:27.656Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-16T22:45:27.656Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-16T22:45:33.046Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config release=mce-config namespace=multicluster-engine status=pending-install description=\"Initial install underway\" values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f global:map[registryOverride:arohcpsvcdev.azurecr.io/acm-d-cache] localCluster:map[addonConfig:map[applicationManagerEnabled:false certPolicyControllerEnabled:false policyControllerEnabled:true searchCollectorEnabled:false] kubeApiUrl:https://kubernetes.default.svc] policy:map[cluster-lifecycle:map[hubconfig:map[replicaCount:2]] grc:map[hubconfig:map[replicaCount:2]]]]\"\ntime=2026-03-16T22:45:33.048Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/6SUMW/bMBCFd/2KgyYbkIJUaVJARYY07RS0Q9OpRQeKvNCMqTuCpBwY8I8vaFmR6lRJkK7P732+x7MvsxjBY+DOSwxwCUpEEUVjccHNPcp4Y0jVIXpDuoBe+iZafJRwg7RXghNykJe/MgCA/Bb9xki8kpI7inkB+dp2IaK3GEuhFFMpmaJna9GXmyo52s5GI3tbiaQNYV7M4bSXpWNr5PaAC+IFxnWvfmeLyckOqRycrSChsUWKtfay7sGHMXuLP8QG2s1jnatku2a6MzoZLEthh9Q/hOP4mNSWG2GnX/J1P5Y6TH6L++arrUMfVuYuTsYe+GEanzT+ZEgZ0s8Vf3lBb+OmB508YjMmBt4RKK32yXqPos/seYb2esBndJa3afb//t3+jXpSakS9leM8O6FF5FlC9vtjtu4a9IQRw5f0pw3ZDh5W6BGiaTFE0TpoMD4gEiyUiJjkRV6dVhfl6Vn57uJHVdXvz+rzDz/zJZycwKznvK6SZ5nt4J4NwdqQujRE6CenhgnGCzM9Lcc3JduB85w+HQctwLHqza+DFGBZF9BiCEInJHuFHprtpLzCIP8EAAD//+XobDEQBQAA\"\ntime=2026-03-16T22:45:33.050Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-16T22:45:33.050Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"error running Helm release deployment Step, failed to deploy helm release: failed post-install: an error on the server (\\\"Internal Server Error: \\\\\\\"/apis/batch/v1/namespaces/multicluster-engine/jobs/finalize-mce-config\\\\\\\": Post \\\\\\\"https://guard.69b883de32e3a9000188c8bc.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \\\\\\\"crypto/rsa: verification error\\\\\\\" while trying to verify candidate authority certificate \\\\\\\"ca\\\\\\\")\\\") has prevented the request from succeeding (delete jobs.batch finalize-mce-config)\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"38239b79b5b799e6cbc8dad16b9ce3a3e5236d97a0d80408b6d601146c6bf4fa","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \\\\\\","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \\\\\\"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["38239b79b5b799e6cbc8dad16b9ce3a3e5236d97a0d80408b6d601146c6bf4fa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033656442757582848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4219,"pr_state":"closed","pr_sha":"ad2d12b8fdba06f94d34e2abe00766c63912193c","final_merged_sha":"8550ece78eb685f5f0357a6937bf6b6e9f457543","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T21:27:34Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"65348f434896f1a8abf0a3f0369128af04d6080c6b4ad5372813a2bb86cdc5e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033656442757582848","occurred_at":"2026-03-16T21:27:34Z","failed_at":"provision","signature_id":"5db5c6dc1ca583042de69025bacdc2c5f167ae8aa7d0d813db05ea5433c172ee","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-velero-image","test_suite":"step graph","failure_text":"time=2026-03-16T21:53:36.176Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image description=\"Step mirror-velero-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e velero.server.registry \u003cnil\u003e}:{\u003cnil\u003e velero.server.repository \u003cnil\u003e}@{\u003cnil\u003e velero.server.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-16T21:53:59.173Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org\\nFetch pull secret for source registry registry.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nFailed to log in to ACR arohcpsvcdev: ERROR: The command failed with an unexpected error. Here is the traceback:\\nERROR: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /oauth2/exchange (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 204, in _new_conn\\n sock = connection.create_connection(\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/util/connection.py\\\", line 60, in create_connection\\n for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/python3.12/socket.py\\\", line 963, in getaddrinfo\\n for res in _socket.getaddrinfo(host, port, family, type, proto, flags):\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\nsocket.gaierror: [Errno -2] Name or service not known\\n\\nThe above exception was the direct cause of the following exception:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 787, in urlopen\\n response = self._make_request(\\n ^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 488, in _make_request\\n raise new_e\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 464, in _make_request\\n self._validate_conn(conn)\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 1093, in _validate_conn\\n conn.connect()\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 759, in connect\\n self.sock = sock = self._new_conn()\\n ^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 211, in _new_conn\\n raise NameResolutionError(self.host, self, e) from e\\nurllib3.exceptions.NameResolutionError: HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\n\\nThe above exception was the direct cause of the following exception:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/adapters.py\\\", line 667, in send\\n resp = conn.urlopen(\\n ^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 841, in urlopen\\n retries = retries.increment(\\n ^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/util/retry.py\\\", line 535, in increment\\n raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\nurllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /oauth2/exchange (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\n\\nDuring handling of the above exception, another exception occurred:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/knack/cli.py\\\", line 233, in invoke\\n cmd_result = self.invocation.execute(args)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 682, in execute\\n raise ex\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 812, in _run_jobs_serially\\n results.append(self._run_job(expanded_arg, cmd_copy))\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 781, in _run_job\\n result = cmd_copy(params)\\n ^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 336, in __call__\\n return self.handler(*args, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/command_operation.py\\\", line 120, in handler\\n return op(**command_args)\\n ^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/custom.py\\\", line 294, in acr_login\\n login_server, _, password = get_login_credentials(\\n ^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 505, in get_login_credentials\\n return _get_credentials(cmd,\\n ^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 449, in _get_credentials\\n return login_server, EMPTY_GUID, _get_aad_token(cli_ctx,\\n ^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 285, in _get_aad_token\\n return _get_aad_token_after_challenge(cli_ctx,\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 159, in _get_aad_token_after_challenge\\n response = requests.post(url=authhost, data=urlencode(content), headers=headers,\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/api.py\\\", line 115, in post\\n return request(\\\"post\\\", url, data=data, json=json, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/api.py\\\", line 59, in request\\n return session.request(method=method, url=url, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/sessions.py\\\", line 589, in request\\n resp = self.send(prep, **send_kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/sessions.py\\\", line 703, in send\\n r = adapter.send(request, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/adapters.py\\\", line 700, in send\\n raise ConnectionError(e, request=request)\\nrequests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /oauth2/exchange (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\nTo check existing issues, please visit: https://github.com/Azure/azure-cli/issues\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9a2a4cf3344f5eab04c519ebc438e0419dbf08c2ef1692b2ed6fd7228e53ded3","failure_pattern":"requests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /oauth2/exchange (Caused by NameResolutionError(\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve","search_query":"requests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /oauth2/exchange (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9a2a4cf3344f5eab04c519ebc438e0419dbf08c2ef1692b2ed6fd7228e53ded3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033654337363775488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4157,"pr_state":"open","pr_sha":"730aedc1e2f03e62758aaba9b43c4ccdd249c551","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T21:19:12Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"1e5fc8af99c14c45ccb0b7569c3765da5eea60083612664387d7d1c837ec2e1e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033654337363775488","occurred_at":"2026-03-16T21:19:12Z","failed_at":"provision","signature_id":"5db5c6dc1ca583042de69025bacdc2c5f167ae8aa7d0d813db05ea5433c172ee","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-velero-image","test_suite":"step graph","failure_text":"time=2026-03-16T22:01:59.625Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image description=\"Step mirror-velero-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e velero.server.registry \u003cnil\u003e}:{\u003cnil\u003e velero.server.repository \u003cnil\u003e}@{\u003cnil\u003e velero.server.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-16T22:02:24.194Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org\\nFetch pull secret for source registry registry.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nFailed to log in to ACR arohcpsvcdev: ERROR: The command failed with an unexpected error. Here is the traceback:\\nERROR: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 204, in _new_conn\\n sock = connection.create_connection(\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/util/connection.py\\\", line 60, in create_connection\\n for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/python3.12/socket.py\\\", line 963, in getaddrinfo\\n for res in _socket.getaddrinfo(host, port, family, type, proto, flags):\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\nsocket.gaierror: [Errno -2] Name or service not known\\n\\nThe above exception was the direct cause of the following exception:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 787, in urlopen\\n response = self._make_request(\\n ^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 488, in _make_request\\n raise new_e\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 464, in _make_request\\n self._validate_conn(conn)\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 1093, in _validate_conn\\n conn.connect()\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 759, in connect\\n self.sock = sock = self._new_conn()\\n ^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 211, in _new_conn\\n raise NameResolutionError(self.host, self, e) from e\\nurllib3.exceptions.NameResolutionError: HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\n\\nThe above exception was the direct cause of the following exception:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/adapters.py\\\", line 667, in send\\n resp = conn.urlopen(\\n ^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 841, in urlopen\\n retries = retries.increment(\\n ^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/util/retry.py\\\", line 535, in increment\\n raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\nurllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\n\\nDuring handling of the above exception, another exception occurred:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/knack/cli.py\\\", line 233, in invoke\\n cmd_result = self.invocation.execute(args)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 682, in execute\\n raise ex\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 812, in _run_jobs_serially\\n results.append(self._run_job(expanded_arg, cmd_copy))\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 781, in _run_job\\n result = cmd_copy(params)\\n ^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 336, in __call__\\n return self.handler(*args, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/command_operation.py\\\", line 120, in handler\\n return op(**command_args)\\n ^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/custom.py\\\", line 294, in acr_login\\n login_server, _, password = get_login_credentials(\\n ^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 505, in get_login_credentials\\n return _get_credentials(cmd,\\n ^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 449, in _get_credentials\\n return login_server, EMPTY_GUID, _get_aad_token(cli_ctx,\\n ^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 276, in _get_aad_token\\n token_params = _handle_challenge_phase(\\n ^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 100, in _handle_challenge_phase\\n challenge = requests.get(request_url, verify=not should_disable_connection_verify())\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/api.py\\\", line 73, in get\\n return request(\\\"get\\\", url, params=params, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/api.py\\\", line 59, in request\\n return session.request(method=method, url=url, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/sessions.py\\\", line 589, in request\\n resp = self.send(prep, **send_kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/sessions.py\\\", line 703, in send\\n r = adapter.send(request, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/adapters.py\\\", line 700, in send\\n raise ConnectionError(e, request=request)\\nrequests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\nTo check existing issues, please visit: https://github.com/Azure/azure-cli/issues\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ee5b1a1a3315ac34a62d3e4a95293e1f5e2f2b1ef572b3ff8359f4b3944b63b3","failure_pattern":"requests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve","search_query":"requests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ee5b1a1a3315ac34a62d3e4a95293e1f5e2f2b1ef572b3ff8359f4b3944b63b3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033653616975286272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4433,"pr_state":"closed","pr_sha":"599dd2c53ef9a79607254436be4a2ed8e0671e8c","final_merged_sha":"599dd2c53ef9a79607254436be4a2ed8e0671e8c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-16T21:16:20Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033653219321712640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4454,"pr_state":"closed","pr_sha":"38f866ad42f9613c0978385116cd5b0c6df796c1","final_merged_sha":"38f866ad42f9613c0978385116cd5b0c6df796c1","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T21:14:45Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"a7debcb8611f674e2d45a01e80c86fe777533bc0ae9831bb910ed992d5e975ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033653219321712640","occurred_at":"2026-03-16T21:14:45Z","failed_at":"provision","signature_id":"a5b5f71a061e60f11c9c511d431eae042d41346983d1442cca50d855704dabbe","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/prometheus","test_suite":"step graph","failure_text":"time=2026-03-16T21:47:01.740Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-03-16T21:47:02.459Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[customResourceState:map[config:map[spec:map[resources:[map[groupVersionKind:map[group:hypershift.openshift.io kind:HostedCluster version:v1beta1] labelsFromPath:map[_id:[spec clusterID] api_url:[spec services [service=APIServer] servicePublishingStrategy route hostname] name:[metadata name] namespace:[metadata namespace]] metricNamePrefix:hostedClusterAPI metrics:[map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=KubeAPIServerAvailable] status]] type:StateSet] help:KubeAPIServer availability condition name:kubeapiserver_available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Available] status]] type:StateSet] help:Cluster availability condition name:available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Progressing] status]] type:StateSet] help:Cluster progressing condition name:progressing]]]]]] create:true enabled:true key:config.yaml name:kube-state-metric-custom-resource-config] image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c] rbac:map[extraRules:[map[apiGroups:[hypershift.openshift.io] resources:[hostedclusters] verbs:[list watch]]]]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:d7e8257f8d8bce64b6df55f81fba92011a6a77269b3350f8b997b152af348dba]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:c0e2dd57-8426-4558-9dfd-5350ad0d2166]] prometheusSpec:map[externalLabels:map[cluster:prow-j1712640-mgmt-1] hcpRemoteWriteUrl:https://msprom-westus3-prow-j1712640-mgmt-1-frzu.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-3693fddb11a14071bdf1db60a24df604/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:dca4b35f73046dc4803c0160c2e6be5a1736a716b26ef35590864d36dfdf950a] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j1712640-mgmt-1-frzu.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-511f0b56c3044481b3f308eb487d8d5d/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-16T21:47:02.459Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T21:47:02.799Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus namespace=prometheus\ntime=2026-03-16T21:47:03.148Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus dryRun=false\ntime=2026-03-16T21:47:03.148Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T21:47:03.768Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus numVersions=0\ntime=2026-03-16T21:47:03.926Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"\\\"arohcp-monitor\\\" has no deployed releases\"\ntime=2026-03-16T21:47:03.926Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T21:47:03.926Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T21:47:04.083Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"failed to get status for release arohcp-monitor: release: not found\"\ntime=2026-03-16T21:47:04.083Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T21:47:04.083Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: \\\"arohcp-monitor\\\" has no deployed releases\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3f7e199865c5445ff3484bb28bdeda70d99439e135094e3f2bd18d66ac18f0c1","failure_pattern":"\"arohcp-monitor\" has no deployed releases","search_query":"error running Helm release deployment Step, failed to deploy helm release: \\\"arohcp-monitor\\\" has no deployed releases"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["3f7e199865c5445ff3484bb28bdeda70d99439e135094e3f2bd18d66ac18f0c1"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033653011489755136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4393,"pr_state":"open","pr_sha":"dcec1a0f0e9fc5822919f091837faed741634449","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T21:13:55Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"05c6bb94497b94587e0dcfac531d9941743be5b00d847bc16f805709eb631182","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033653011489755136","occurred_at":"2026-03-16T21:13:55Z","failed_at":"provision","signature_id":"a5b5f71a061e60f11c9c511d431eae042d41346983d1442cca50d855704dabbe","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/prometheus","test_suite":"step graph","failure_text":"time=2026-03-16T21:45:19.279Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-03-16T21:45:19.891Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[customResourceState:map[config:map[spec:map[resources:[map[groupVersionKind:map[group:hypershift.openshift.io kind:HostedCluster version:v1beta1] labelsFromPath:map[_id:[spec clusterID] api_url:[spec services [service=APIServer] servicePublishingStrategy route hostname] name:[metadata name] namespace:[metadata namespace]] metricNamePrefix:hostedClusterAPI metrics:[map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=KubeAPIServerAvailable] status]] type:StateSet] help:KubeAPIServer availability condition name:kubeapiserver_available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Available] status]] type:StateSet] help:Cluster availability condition name:available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Progressing] status]] type:StateSet] help:Cluster progressing condition name:progressing]]]]]] create:true enabled:true key:config.yaml name:kube-state-metric-custom-resource-config] image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c] rbac:map[extraRules:[map[apiGroups:[hypershift.openshift.io] resources:[hostedclusters] verbs:[list watch]]]]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:d7e8257f8d8bce64b6df55f81fba92011a6a77269b3350f8b997b152af348dba]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:00d715af-ce8d-440f-8c1b-630fda34474f]] prometheusSpec:map[externalLabels:map[cluster:prow-j9755136-mgmt-1] hcpRemoteWriteUrl:https://msprom-westus3-prow-j9755136-mgmt-1-77iw.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-8392ee99bb7b4c47be651b9c93097b71/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:dca4b35f73046dc4803c0160c2e6be5a1736a716b26ef35590864d36dfdf950a] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j9755136-mgmt-1-77iw.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-ce8065f976124c569ca5653ccdbdf2a0/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-16T21:45:19.891Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T21:45:20.256Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus namespace=prometheus\ntime=2026-03-16T21:45:20.682Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus dryRun=false\ntime=2026-03-16T21:45:20.682Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T21:45:20.833Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T21:47:28.992Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"failed post-install: Unauthorized\"\ntime=2026-03-16T21:47:28.992Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T21:47:28.992Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T21:47:45.274Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus release=arohcp-monitor namespace=prometheus status=failed description=\"Release \\\"arohcp-monitor\\\" failed: failed post-install: Unauthorized\" values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[customResourceState:map[config:map[spec:map[resources:[map[groupVersionKind:map[group:hypershift.openshift.io kind:HostedCluster version:v1beta1] labelsFromPath:map[_id:[spec clusterID] api_url:[spec services [service=APIServer] servicePublishingStrategy route hostname] name:[metadata name] namespace:[metadata namespace]] metricNamePrefix:hostedClusterAPI metrics:[map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=KubeAPIServerAvailable] status]] type:StateSet] help:KubeAPIServer availability condition name:kubeapiserver_available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Available] status]] type:StateSet] help:Cluster availability condition name:available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Progressing] status]] type:StateSet] help:Cluster progressing condition name:progressing]]]]]] create:true enabled:true key:config.yaml name:kube-state-metric-custom-resource-config] image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c] rbac:map[extraRules:[map[apiGroups:[hypershift.openshift.io] resources:[hostedclusters] verbs:[list watch]]]]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:d7e8257f8d8bce64b6df55f81fba92011a6a77269b3350f8b997b152af348dba]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:00d715af-ce8d-440f-8c1b-630fda34474f]] prometheusSpec:map[externalLabels:map[cluster:prow-j9755136-mgmt-1] hcpRemoteWriteUrl:https://msprom-westus3-prow-j9755136-mgmt-1-77iw.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-8392ee99bb7b4c47be651b9c93097b71/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:dca4b35f73046dc4803c0160c2e6be5a1736a716b26ef35590864d36dfdf950a] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j9755136-mgmt-1-77iw.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-ce8065f976124c569ca5653ccdbdf2a0/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-16T21:47:45.275Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/6yWTW/aTBDH7/4UK5+C5EUJz9M0osqhjXqqUlVp1UqpeljvTs0G7441MwYh8eErYwIuhAYZjjv7n9/OwLw4KUEUAWNNFljdKmfEiMlLuMD8Cax88tGNWcjHIlOt6bMJsDHBDOLKwpWxz+bBz0QppdK7smYBesAS0kylhnBiKx0wekHS0zoHzWIEdAAhb7kRpdmLvhVhAJlAzRorICNIB9QffHQ+Fmdw6mq/oLtv497XdE9r9cH7f6sf6r1sTQkkwURTAA2pLqEPZHrDre/QYhTjI5AOEJAW2ho7gfMiic8cI89NdV7iHGnqY6EZROcL6fejNvXbORcQgUzZ/0/a4UV0oAksUlOaJ2EpggDrTaOfhmFBMkWvoulAFiwQtKk8A81gr6360HowXuvNr0Azb6HT+0dNsWNAB6bTQdf31mId5fQYzkM4Ovz7Woz4WPyAfII4vcP42xc1GfEYd4edC555bX/2/25K704itA737RB5NXNtaxYMm2ZZ3dkV4uUE/8YHswExSBM2r71Dq2hfbAs2+fUu2Zbxx2aZcrJU8wkQKPGheT1UKgeZA0R14YxAY75IR5eja335n766/ja6Gv//Zjy6fEwHajhUBzVvx6Obx3QwSJbqCX1UUx/drY8RqPMJgFFtN3935e/u+mSpKsLmdhtopip0rfg4SKZKLDIVgNkUDRLJAal80UneAds/AQAA//+NZhXbqAgAAA==\"\ntime=2026-03-16T21:47:45.275Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T21:47:45.276Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: failed post-install: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82e6580bc32e56aa7378422fd2960d02a2eebbb9fbfbc08994384c40137bbab6","failure_pattern":"failed post-install: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed post-install: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["82e6580bc32e56aa7378422fd2960d02a2eebbb9fbfbc08994384c40137bbab6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4483/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033650201863917568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4483,"pr_state":"open","pr_sha":"c5356b74da140f99e74e5c7493d26cb8dff9fd87","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T21:02:46Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"9f3112de835ef2d2645589e627eb6d1f1b474ba0bb91a4577677c3451d8d7ab4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4483/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033650201863917568","occurred_at":"2026-03-16T21:02:46Z","failed_at":"provision","signature_id":"7dc4fb9cfe9e0a6ce7afe61cb508e1ad4e8fa4df483ce758063f5f3f64cd897c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RouteMonitorOperator/management/default-ingresscontroller-hack","test_suite":"step graph","failure_text":"time=2026-03-16T21:37:32.736Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=default-ingresscontroller-hack description=\"Step default-ingresscontroller-hack\\n Kind: Shell\\n Command: make default-ingresscontroller-hack\\n\"\ntime=2026-03-16T21:37:34.160Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=default-ingresscontroller-hack err=\"failed to prepare kubeconfig: failed to ensure cluster admin role: failed to get current user object ID: failed to get current user: /me request is only valid with delegated authentication flow.\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1466406c48902bc9d663394fbe43cd72c554c24ca2f6aa7ec287ef1819329804","failure_pattern":"failed to prepare kubeconfig: failed to ensure cluster admin role: failed to get current user object ID: failed to get current user: /me request is only valid with delegated authentication flow.","search_query":"failed to prepare kubeconfig: failed to ensure cluster admin role: failed to get current user object ID: failed to get current user: /me request is only valid with delegated authentication flow."}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1466406c48902bc9d663394fbe43cd72c554c24ca2f6aa7ec287ef1819329804"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033644310720679936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4485,"pr_state":"closed","pr_sha":"7e50507656e9445534b8f8c382c77a0d23edeed7","final_merged_sha":"56bf314caa75ae5340defeb47d3ce32c4afb8aea","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T20:39:21Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"9811a55f8f2c1e716bcbb616daddfc1ac46bae1ae57f8db6a10c8d2582885946","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033644310720679936","occurred_at":"2026-03-16T20:39:21Z","failed_at":"provision","signature_id":"af99b9283742f286e546fac9212a5301d331519de67f18f8db49375b0ce9e2c0","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-policies","test_suite":"step graph","failure_text":"time=2026-03-16T21:18:18.396Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies description=\"Step deploy-policies\\n Kind: Helm\\n\"\ntime=2026-03-16T21:18:19.181Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies values=map[]\ntime=2026-03-16T21:18:19.181Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies\ntime=2026-03-16T21:18:19.367Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies namespace=open-cluster-management-policies\ntime=2026-03-16T21:18:19.647Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies dryRun=false\ntime=2026-03-16T21:18:19.647Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies\ntime=2026-03-16T21:18:19.757Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies\ntime=2026-03-16T21:18:22.140Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies err=\"unable to continue with install: could not get information about the resource Policy \\\"ingress-controller-check\\\" in namespace \\\"open-cluster-management-policies\\\": an error on the server (\\\"Internal Server Error: \\\\\\\"/apis/policy.open-cluster-management.io/v1/namespaces/open-cluster-management-policies/policies/ingress-controller-check\\\\\\\": Post \\\\\\\"https://guard.69b86fad54541c00013118c8.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \\\\\\\"crypto/rsa: verification error\\\\\\\" while trying to verify candidate authority certificate \\\\\\\"ca\\\\\\\")\\\") has prevented the request from succeeding (get policies.policy.open-cluster-management.io ingress-controller-check)\"\ntime=2026-03-16T21:18:22.140Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies\ntime=2026-03-16T21:18:22.140Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies\ntime=2026-03-16T21:18:22.250Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies err=\"failed to get status for release policy: release: not found\"\ntime=2026-03-16T21:18:22.250Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies\ntime=2026-03-16T21:18:22.250Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies err=\"error running Helm release deployment Step, failed to deploy helm release: unable to continue with install: could not get information about the resource Policy \\\"ingress-controller-check\\\" in namespace \\\"open-cluster-management-policies\\\": an error on the server (\\\"Internal Server Error: \\\\\\\"/apis/policy.open-cluster-management.io/v1/namespaces/open-cluster-management-policies/policies/ingress-controller-check\\\\\\\": Post \\\\\\\"https://guard.69b86fad54541c00013118c8.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \\\\\\\"crypto/rsa: verification error\\\\\\\" while trying to verify candidate authority certificate \\\\\\\"ca\\\\\\\")\\\") has prevented the request from succeeding (get policies.policy.open-cluster-management.io ingress-controller-check)\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"38239b79b5b799e6cbc8dad16b9ce3a3e5236d97a0d80408b6d601146c6bf4fa","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \\\\\\","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \\\\\\"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["38239b79b5b799e6cbc8dad16b9ce3a3e5236d97a0d80408b6d601146c6bf4fa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4484/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033644034940997632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4484,"pr_state":"closed","pr_sha":"0f991f023fc17b4d35b5f4517d743f35d4c10d3c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T20:38:15Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"6fe8da7569f5298b29804dd54366e174df4aa45308f6d5ea9ca0c50058cba705","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4484/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033644034940997632","occurred_at":"2026-03-16T20:38:15Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033641637262921728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4467,"pr_state":"closed","pr_sha":"f1bd32597744d7875c4634181a1949fcc0c3216f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T20:28:44Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"23a44a17bcc68260e627c8af63e5c74569b1241bcf796e689f3aa51409af8720","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033641637262921728","occurred_at":"2026-03-16T20:28:44Z","failed_at":"provision","signature_id":"692850ccec3627f6f18c0069e48528a9b9f4e27dd84dcc0a7a087aa7c2d7afa7","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce","test_suite":"step graph","failure_text":"time=2026-03-16T21:05:37.584Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce description=\"Step deploy-mce\\n Kind: Helm\\n\"\ntime=2026-03-16T21:05:38.143Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f imageRegistry:arohcpsvcdev.azurecr.io imageRootRepository:acm-d-cache]\"\ntime=2026-03-16T21:05:38.144Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-16T21:05:38.255Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce namespace=multicluster-engine\ntime=2026-03-16T21:05:38.365Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce dryRun=false\ntime=2026-03-16T21:05:38.365Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-16T21:05:38.473Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-16T21:08:00.581Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce err=\"failed post-install: an error on the server (\\\"Internal Server Error: \\\\\\\"/apis/batch/v1/namespaces/multicluster-engine/jobs/finalize-mce\\\\\\\": Post \\\\\\\"https://guard.69b86d9039445e00019c3745.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": proxy error from localhost:9443 while dialing guard.69b86d9039445e00019c3745.svc.cluster.local:443, code 503: 503 Service Unavailable\\\") has prevented the request from succeeding (delete jobs.batch finalize-mce)\"\ntime=2026-03-16T21:08:00.581Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-16T21:08:00.581Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-16T21:08:01.250Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce err=\"failed to get status for release mce: kubernetes cluster unreachable: the server has asked for the client to provide credentials\"\ntime=2026-03-16T21:08:01.250Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-16T21:08:01.250Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce err=\"error running Helm release deployment Step, failed to deploy helm release: failed post-install: an error on the server (\\\"Internal Server Error: \\\\\\\"/apis/batch/v1/namespaces/multicluster-engine/jobs/finalize-mce\\\\\\\": Post \\\\\\\"https://guard.69b86d9039445e00019c3745.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": proxy error from localhost:9443 while dialing guard.69b86d9039445e00019c3745.svc.cluster.local:443, code 503: 503 Service Unavailable\\\") has prevented the request from succeeding (delete jobs.batch finalize-mce)\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2fb7602e309cfda4b6a96b15fce468cded201bc29b139d91d24914f4383d7dc1","failure_pattern":"failed post-install: an error on the server (\"Internal Server Error: \\\\\"/apis/batch/v1/namespaces/multicluster-engine/jobs/finalize-mce\\\\\": Post \\\\\"\u003curl\u003e\": proxy error from localhost:9443 while dialing guard.69b86d9039445e00019c3745.svc.cluster.local:443","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed post-install: an error on the server (\\\"Internal Server Error: \\\\\\\"/apis/batch/v1/namespaces/multicluster-engine/jobs/finalize-mce\\\\\\\": Post \\\\\\\"https://guard.69b86d9039445e00019c3745.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": proxy error from localhost:9443 while dialing guard.69b86d9039445e00019c3745.svc.cluster.local:443, code 503: 503 Service Unavailable\\\") has prevented the request from succeeding (delete jobs.batch finalize-mce)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2fb7602e309cfda4b6a96b15fce468cded201bc29b139d91d24914f4383d7dc1"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033640927116922880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4226,"pr_state":"closed","pr_sha":"138964dfc1d79a030eb9c07890259cb904e592a1","final_merged_sha":"138964dfc1d79a030eb9c07890259cb904e592a1","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T20:25:42Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c5498cec88778f5af3110a63a2291eda70751a40bd73891e86c6b4b9219ac40f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033640927116922880","occurred_at":"2026-03-16T20:25:42Z","failed_at":"provision","signature_id":"a5b5f71a061e60f11c9c511d431eae042d41346983d1442cca50d855704dabbe","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/prometheus","test_suite":"step graph","failure_text":"time=2026-03-16T21:01:24.308Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-03-16T21:01:24.970Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[customResourceState:map[config:map[spec:map[resources:[map[groupVersionKind:map[group:hypershift.openshift.io kind:HostedCluster version:v1beta1] labelsFromPath:map[_id:[spec clusterID] api_url:[spec services [service=APIServer] servicePublishingStrategy route hostname] name:[metadata name] namespace:[metadata namespace]] metricNamePrefix:hostedClusterAPI metrics:[map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=KubeAPIServerAvailable] status]] type:StateSet] help:KubeAPIServer availability condition name:kubeapiserver_available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Available] status]] type:StateSet] help:Cluster availability condition name:available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Progressing] status]] type:StateSet] help:Cluster progressing condition name:progressing]]]]]] create:true enabled:true key:config.yaml name:kube-state-metric-custom-resource-config] image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c] rbac:map[extraRules:[map[apiGroups:[hypershift.openshift.io] resources:[hostedclusters] verbs:[list watch]]]]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:d7e8257f8d8bce64b6df55f81fba92011a6a77269b3350f8b997b152af348dba]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:0c3e5218-5ebd-4cb6-aca0-9a4ce3c51e56]] prometheusSpec:map[externalLabels:map[cluster:prow-j6922880-mgmt-1] hcpRemoteWriteUrl:https://msprom-westus3-prow-j6922880-mgmt-1-hrjs.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-a6c994f4b3b644e9a187298c90941783/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:dca4b35f73046dc4803c0160c2e6be5a1736a716b26ef35590864d36dfdf950a] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j6922880-mgmt-1-hrjs.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-1cd0089dd64c44c8a8571eaf7bd7f61d/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-16T21:01:24.970Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T21:01:25.281Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus namespace=prometheus\ntime=2026-03-16T21:01:25.615Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus dryRun=false\ntime=2026-03-16T21:01:25.615Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T21:01:25.781Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T21:02:18.525Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"failed post-install: Unauthorized\"\ntime=2026-03-16T21:02:18.525Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T21:02:18.525Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T21:02:32.544Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus release=arohcp-monitor namespace=prometheus status=failed description=\"Release \\\"arohcp-monitor\\\" failed: failed post-install: Unauthorized\" values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[customResourceState:map[config:map[spec:map[resources:[map[groupVersionKind:map[group:hypershift.openshift.io kind:HostedCluster version:v1beta1] labelsFromPath:map[_id:[spec clusterID] api_url:[spec services [service=APIServer] servicePublishingStrategy route hostname] name:[metadata name] namespace:[metadata namespace]] metricNamePrefix:hostedClusterAPI metrics:[map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=KubeAPIServerAvailable] status]] type:StateSet] help:KubeAPIServer availability condition name:kubeapiserver_available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Available] status]] type:StateSet] help:Cluster availability condition name:available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Progressing] status]] type:StateSet] help:Cluster progressing condition name:progressing]]]]]] create:true enabled:true key:config.yaml name:kube-state-metric-custom-resource-config] image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c] rbac:map[extraRules:[map[apiGroups:[hypershift.openshift.io] resources:[hostedclusters] verbs:[list watch]]]]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:d7e8257f8d8bce64b6df55f81fba92011a6a77269b3350f8b997b152af348dba]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:0c3e5218-5ebd-4cb6-aca0-9a4ce3c51e56]] prometheusSpec:map[externalLabels:map[cluster:prow-j6922880-mgmt-1] hcpRemoteWriteUrl:https://msprom-westus3-prow-j6922880-mgmt-1-hrjs.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-a6c994f4b3b644e9a187298c90941783/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:dca4b35f73046dc4803c0160c2e6be5a1736a716b26ef35590864d36dfdf950a] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j6922880-mgmt-1-hrjs.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-1cd0089dd64c44c8a8571eaf7bd7f61d/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-16T21:02:32.545Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/6xWTW8TMRC976+wcmqkOGqCqKqgHqBwQkWoIJCKOHjtx8bN2mONZxsi5cejZPuxDYSmSY/79s2bZ489nqKGKEamhi2yOlPOiBFT1jii8hpWPvroJlnYx2qgWuiTCbiHcIO4RnIy9g7u/yiUUqr3mdwFRS/EvYHqJaYAmaLJm1+37HvksqnxmKNNDZZgoqnAQ25q7CMyO81t7NBSFOMjWNvU6CabCjrDUnRZC4mpX0g9IBAvtDV2ipeV5PxSO3ArmOcmHaSYyGmaR/xV7F1UmhK6810hgk29f6WbEhwhyNqkdKDC/f04TCYLsan2OgYdkUUWBG2Sz+CbvfY6koPG70QsB9ylRyr7xkfInHi2Z/j/vH8B33iLTvsxTFObdGiR9sRlMQIdIOztzkIdF5TA5h/trQ09r5ss4Etq3e9kYEvslpxPsreQ3vnofKye7eqbqb0z4mP1HeWUaHZO8ZevGjbiKW62bBd8zrf4xna+tZaaKIfXpSO0e13Wpi/admeCuUukM2S1tqztmhFaRutofe82kj8r63ukmhYBB636scjTuYufb4qH7vFh9VjnYqnmUzCU+IAsJiRVQuZAVEfOCFbwUW98PD7Rx6/06OTreDQ5Hk3Gr696fTUcqq2c8WR0etXr94uluiYf1cxHd+ZjBHdGDIrqYbLojhSbs0SxVIlp9ffB6EAlci15N5GBqqkaqIC8euKLpSJ2YFUuOot3yPZPAAAA//9TChsdCAkAAA==\"\ntime=2026-03-16T21:02:32.545Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T21:02:32.546Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: failed post-install: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82e6580bc32e56aa7378422fd2960d02a2eebbb9fbfbc08994384c40137bbab6","failure_pattern":"failed post-install: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed post-install: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["82e6580bc32e56aa7378422fd2960d02a2eebbb9fbfbc08994384c40137bbab6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033639864724885504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4224,"pr_state":"closed","pr_sha":"89dbea9535e9da4e251791f532920158951c59f0","final_merged_sha":"0d2c310eee81cd65f33fd842930856c19dc160e3","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T20:21:40Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"5004ec13afe4c94f6dbdede8741fb18611f2b5db65de0d88e5dde472926aa157","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033639864724885504","occurred_at":"2026-03-16T20:21:40Z","failed_at":"provision","signature_id":"1afb28c8e157da381239a18ffe2b4a0af4e45e4c32936e2ba90253d7fe3da81c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RouteMonitorOperator/global/mirror-blackbox-image","test_suite":"step graph","failure_text":"time=2026-03-16T21:01:18.829Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=global step=mirror-blackbox-image description=\"Step mirror-blackbox-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e routeMonitorOperator.blackboxExporterImage.registry \u003cnil\u003e}:{\u003cnil\u003e routeMonitorOperator.blackboxExporterImage.repository \u003cnil\u003e}@{\u003cnil\u003e routeMonitorOperator.blackboxExporterImage.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-16T21:01:41.643Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=global step=mirror-blackbox-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org\\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nFailed to log in to ACR arohcpsvcdev: ERROR: The command failed with an unexpected error. Here is the traceback:\\nERROR: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /oauth2/exchange (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 204, in _new_conn\\n sock = connection.create_connection(\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/util/connection.py\\\", line 60, in create_connection\\n for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/python3.12/socket.py\\\", line 963, in getaddrinfo\\n for res in _socket.getaddrinfo(host, port, family, type, proto, flags):\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\nsocket.gaierror: [Errno -2] Name or service not known\\n\\nThe above exception was the direct cause of the following exception:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 787, in urlopen\\n response = self._make_request(\\n ^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 488, in _make_request\\n raise new_e\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 464, in _make_request\\n self._validate_conn(conn)\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 1093, in _validate_conn\\n conn.connect()\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 759, in connect\\n self.sock = sock = self._new_conn()\\n ^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 211, in _new_conn\\n raise NameResolutionError(self.host, self, e) from e\\nurllib3.exceptions.NameResolutionError: HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\n\\nThe above exception was the direct cause of the following exception:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/adapters.py\\\", line 667, in send\\n resp = conn.urlopen(\\n ^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 841, in urlopen\\n retries = retries.increment(\\n ^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/util/retry.py\\\", line 535, in increment\\n raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\nurllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /oauth2/exchange (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\n\\nDuring handling of the above exception, another exception occurred:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/knack/cli.py\\\", line 233, in invoke\\n cmd_result = self.invocation.execute(args)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 682, in execute\\n raise ex\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 812, in _run_jobs_serially\\n results.append(self._run_job(expanded_arg, cmd_copy))\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 781, in _run_job\\n result = cmd_copy(params)\\n ^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 336, in __call__\\n return self.handler(*args, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/command_operation.py\\\", line 120, in handler\\n return op(**command_args)\\n ^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/custom.py\\\", line 294, in acr_login\\n login_server, _, password = get_login_credentials(\\n ^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 505, in get_login_credentials\\n return _get_credentials(cmd,\\n ^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 449, in _get_credentials\\n return login_server, EMPTY_GUID, _get_aad_token(cli_ctx,\\n ^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 285, in _get_aad_token\\n return _get_aad_token_after_challenge(cli_ctx,\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 159, in _get_aad_token_after_challenge\\n response = requests.post(url=authhost, data=urlencode(content), headers=headers,\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/api.py\\\", line 115, in post\\n return request(\\\"post\\\", url, data=data, json=json, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/api.py\\\", line 59, in request\\n return session.request(method=method, url=url, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/sessions.py\\\", line 589, in request\\n resp = self.send(prep, **send_kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/sessions.py\\\", line 703, in send\\n r = adapter.send(request, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/adapters.py\\\", line 700, in send\\n raise ConnectionError(e, request=request)\\nrequests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /oauth2/exchange (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\nTo check existing issues, please visit: https://github.com/Azure/azure-cli/issues\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9a2a4cf3344f5eab04c519ebc438e0419dbf08c2ef1692b2ed6fd7228e53ded3","failure_pattern":"requests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /oauth2/exchange (Caused by NameResolutionError(\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve","search_query":"requests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /oauth2/exchange (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9a2a4cf3344f5eab04c519ebc438e0419dbf08c2ef1692b2ed6fd7228e53ded3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033639579705151488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4219,"pr_state":"closed","pr_sha":"ad2d12b8fdba06f94d34e2abe00766c63912193c","final_merged_sha":"8550ece78eb685f5f0357a6937bf6b6e9f457543","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T20:20:33Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"bc738d64d8490c5c4ac5bf554e0d286092d09412c0fac350173f84ce3f27eb5b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033639579705151488","occurred_at":"2026-03-16T20:20:33Z","failed_at":"provision","signature_id":"5ac3f50227d6a7076813d8bbbe9de0e8c8876a26e6800117bb2654c1677cfb10","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/acrpull","test_suite":"step graph","failure_text":"time=2026-03-16T21:01:50.368Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull description=\"Step acrpull\\n Kind: Helm\\n\"\ntime=2026-03-16T21:01:51.040Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull values=\"map[image:mcr.microsoft.com/aks/msi-acrpull@sha256:042511defb0842f02593e48e68d83a20a1af4265db4e23fca484356e34a6c883 namespace:acrpull replicas:2 ttlRotationFraction:0.5]\"\ntime=2026-03-16T21:01:51.040Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull\ntime=2026-03-16T21:01:51.336Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull namespace=acrpull\ntime=2026-03-16T21:01:51.639Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull dryRun=false\ntime=2026-03-16T21:01:51.639Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull\ntime=2026-03-16T21:01:51.761Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull\ntime=2026-03-16T21:01:59.535Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull err=Unauthorized\ntime=2026-03-16T21:01:59.536Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull\ntime=2026-03-16T21:01:59.536Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull\ntime=2026-03-16T21:02:02.487Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull release=acrpull namespace=acrpull status=failed description=\"Release \\\"acrpull\\\" failed: Unauthorized\" values=\"map[image:mcr.microsoft.com/aks/msi-acrpull@sha256:042511defb0842f02593e48e68d83a20a1af4265db4e23fca484356e34a6c883 namespace:acrpull replicas:2 ttlRotationFraction:0.5]\"\ntime=2026-03-16T21:02:02.488Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5yTT2vbQBDF7/oUg042WMZ2aaAuOaR/TqUlpKWHlB5WuxN14t0ZdXeUEPCHL7KceiscN+Rmz775zRsxr/CoEDFJFy0mOAdn1KipPU6kvkWrn4jdOmkkbmYwlL6YgH9LeIe8q6TW2Mfy9EcBAFB+xXhHFi+slY61nEFpbGw77//5OWjfd0klXO2tfMAbYlISzqQ1sSNu0nz/fx7IRklyo3MroRe+BBYSVf8H+i4pxivx+G5ozFCVFdYo3mOs6sPrY2/f9ITao3EYK/RoR/by5tMTR4zcwYj13XhyRombCxcoJRK+FE/2IRuQgRPaiFqFTs0O3O60xzbMvs5xk7n4UtxnYVKJpw/iSbcjmyob5Cri7w6T7k3m857LOb7uy0i7s6/McPdHkcXPt8WmqzEyKqaPfYpSsYX7XxgRlAImNaGFGvUekWHijGJfnpSrxeqsWryqlmffVsv1Yrl+vbwupzCfwynNm+tyOi22cCvEsCF258SMMcu+MBwin2d9HPJiC22U/vVgdAatuEH8PMgMvDQzCJiSaXqkRIcR6odseYfJ/gkAAP//JisWsaEEAAA=\"\ntime=2026-03-16T21:02:02.489Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull\ntime=2026-03-16T21:02:02.489Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull err=\"error running Helm release deployment Step, failed to deploy helm release: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3f5a2d82a327ba9cb1fbefb6d8dd1ec1c24c537d00d9bb445cff295c67dc36e2","failure_pattern":"Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["3f5a2d82a327ba9cb1fbefb6d8dd1ec1c24c537d00d9bb445cff295c67dc36e2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4466/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033632571971080192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4466,"pr_state":"closed","pr_sha":"8cb4bd5f1213fd23ab6fcccc158048912ff9661e","final_merged_sha":"8cb4bd5f1213fd23ab6fcccc158048912ff9661e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-16T19:52:40Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033632295461588992","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4468,"pr_state":"closed","pr_sha":"10f6db82fe4e37a551e5388b2dc0840ec0886921","final_merged_sha":"532823b3d4dd565d5b0521f892350ccadb788955","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T19:51:36Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"66aeb02e7936ad1307bc459a82217995e7aee9f17ee8944ad9b37b2b7432e907","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033632295461588992","occurred_at":"2026-03-16T19:51:36Z","failed_at":"provision","signature_id":"5058d0fd74c81872666f058ed001e141a7fb54cc9c8248d41819c9fdda7930a5","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RouteMonitorOperator/management/deploy","test_suite":"step graph","failure_text":"time=2026-03-16T20:16:22.096Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-16T20:16:22.596Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy values=\"map[imageDigestBlackbox:sha256:b04a9fef4fa086a02fc7fcd8dcdbc4b7b35cc30cdee860fdc6a19dd8b208d63e imageDigestManager:sha256:5e94ff37bcb8c1ebebce0c7c1f33cef8e7cf163f1b0baf5b50c9b771ca112846 imageDigestOperator:sha256:5e94ff37bcb8c1ebebce0c7c1f33cef8e7cf163f1b0baf5b50c9b771ca112846 imageRegistryBlackbox:arohcpsvcdev.azurecr.io imageRegistryManager:arohcpsvcdev.azurecr.io imageRegistryOperator:arohcpsvcdev.azurecr.io imageRepositoryBlackbox:prometheus/blackbox-exporter imageRepositoryManager:app-sre/route-monitor-operator imageRepositoryOperator:app-sre/route-monitor-operator]\"\ntime=2026-03-16T20:16:22.596Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy\ntime=2026-03-16T20:16:23.346Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace openshift-route-monitor-operator: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a36b175ede41d233d6b695b53eeb0202c33fa8abba1121a12200be20e321df3a","failure_pattern":"failed to apply namespace openshift-route-monitor-operator: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace openshift-route-monitor-operator: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a36b175ede41d233d6b695b53eeb0202c33fa8abba1121a12200be20e321df3a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033630407211094016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4482,"pr_state":"closed","pr_sha":"0b90840021e1fdc816441a8107f7b8aa42d1b0ed","final_merged_sha":"04f43f02c3600758a856fe0f5464f9937b913cdd","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T19:44:06Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"fa6684717f10e9cbac77fa8a92cf08030958b83ee969e9db8c99eeb55c2ca4b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033630407211094016","occurred_at":"2026-03-16T19:44:06Z","failed_at":"provision","signature_id":"692850ccec3627f6f18c0069e48528a9b9f4e27dd84dcc0a7a087aa7c2d7afa7","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce","test_suite":"step graph","failure_text":"time=2026-03-16T20:26:47.029Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce description=\"Step deploy-mce\\n Kind: Helm\\n\"\ntime=2026-03-16T20:26:47.648Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f imageRegistry:arohcpsvcdev.azurecr.io imageRootRepository:acm-d-cache]\"\ntime=2026-03-16T20:26:47.648Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-16T20:26:47.707Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce namespace=multicluster-engine\ntime=2026-03-16T20:26:47.819Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce dryRun=false\ntime=2026-03-16T20:26:47.819Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-16T20:26:47.932Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-16T20:28:26.817Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce err=\"failed post-install: warning: Hook post-install multicluster-engine/templates/finalize-mce.job.yaml failed: an error on the server (\\\"Internal Server Error: \\\\\\\"/apis/batch/v1/namespaces/multicluster-engine/jobs/finalize-mce?fieldManager=templatize\u0026amp;fieldValidation=Strict\u0026amp;force=false\\\\\\\": Post \\\\\\\"https://guard.69b863f2318b900001096102.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": proxy error from localhost:9443 while dialing guard.69b863f2318b900001096102.svc.cluster.local:443, code 503: 503 Service Unavailable\\\") has prevented the request from succeeding (patch jobs.batch finalize-mce)\"\ntime=2026-03-16T20:28:26.817Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-16T20:28:26.817Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-16T20:28:26.961Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce err=\"failed to get status for release mce: query: failed to query with labels: an error on the server (\\\"Internal Server Error: \\\\\\\"/api/v1/namespaces/multicluster-engine/secrets?labelSelector=name%3Dmce%2Cowner%3Dhelm\\\\\\\": Post \\\\\\\"https://guard.69b863f2318b900001096102.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": proxy error from localhost:9443 while dialing guard.69b863f2318b900001096102.svc.cluster.local:443, code 503: 503 Service Unavailable\\\") has prevented the request from succeeding (get secrets)\"\ntime=2026-03-16T20:28:26.961Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-16T20:28:26.961Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce err=\"error running Helm release deployment Step, failed to deploy helm release: failed post-install: warning: Hook post-install multicluster-engine/templates/finalize-mce.job.yaml failed: an error on the server (\\\"Internal Server Error: \\\\\\\"/apis/batch/v1/namespaces/multicluster-engine/jobs/finalize-mce?fieldManager=templatize\u0026amp;fieldValidation=Strict\u0026amp;force=false\\\\\\\": Post \\\\\\\"https://guard.69b863f2318b900001096102.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": proxy error from localhost:9443 while dialing guard.69b863f2318b900001096102.svc.cluster.local:443, code 503: 503 Service Unavailable\\\") has prevented the request from succeeding (patch jobs.batch finalize-mce)\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4aea89b4d0ce555395f855fb478cc90da3a000e0a9cc055624b21c1b2b09b0a","failure_pattern":"failed post-install: warning: Hook post-install multicluster-engine/templates/finalize-mce.job.yaml failed: an error on the server (\"Internal Server Error","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed post-install: warning: Hook post-install multicluster-engine/templates/finalize-mce.job.yaml failed: an error on the server (\\\"Internal Server Error: \\\\\\\"/apis/batch/v1/namespaces/multicluster-engine/jobs/finalize-mce?fieldManager=templatize\u0026amp;fieldValidation=Strict\u0026amp;force=false\\\\\\\": Post \\\\\\\"https://guard.69b863f2318b900001096102.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": proxy error from localhost:9443 while dialing guard.69b863f2318b900001096102.svc.cluster.local:443, code 503: 503 Service Unavailable\\\") has prevented the request from succeeding (patch jobs.batch finalize-mce)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f4aea89b4d0ce555395f855fb478cc90da3a000e0a9cc055624b21c1b2b09b0a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4449/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033629326498336768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4449,"pr_state":"closed","pr_sha":"205984e6ef0d1b76b1e624468408854ec2a65a8c","final_merged_sha":"205984e6ef0d1b76b1e624468408854ec2a65a8c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T19:39:49Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"0adab95cf58ea90d6a00bddc259890efd93d48c2f4c56c17a111b5d1968f8f1b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4449/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033629326498336768","occurred_at":"2026-03-16T19:39:49Z","failed_at":"provision","signature_id":"031c54fa35aec9e0c04cce1b936b80aba069b2956505852e58193ab6a029a41c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RouteMonitorOperator/management/deploy","test_suite":"step graph","failure_text":"time=2026-03-16T20:05:53.635Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-16T20:05:54.216Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy values=\"map[imageDigestBlackbox:sha256:b04a9fef4fa086a02fc7fcd8dcdbc4b7b35cc30cdee860fdc6a19dd8b208d63e imageDigestManager:sha256:5e94ff37bcb8c1ebebce0c7c1f33cef8e7cf163f1b0baf5b50c9b771ca112846 imageDigestOperator:sha256:5e94ff37bcb8c1ebebce0c7c1f33cef8e7cf163f1b0baf5b50c9b771ca112846 imageRegistryBlackbox:arohcpsvcdev.azurecr.io imageRegistryManager:arohcpsvcdev.azurecr.io imageRegistryOperator:arohcpsvcdev.azurecr.io imageRepositoryBlackbox:prometheus/blackbox-exporter imageRepositoryManager:app-sre/route-monitor-operator imageRepositoryOperator:app-sre/route-monitor-operator]\"\ntime=2026-03-16T20:05:54.216Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy\ntime=2026-03-16T20:05:54.534Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy namespace=openshift-route-monitor-operator\ntime=2026-03-16T20:05:54.815Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy dryRun=false\ntime=2026-03-16T20:05:54.815Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy\ntime=2026-03-16T20:05:54.921Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy\ntime=2026-03-16T20:05:58.181Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy err=\"unable to continue with install: could not get information about the resource ClusterRole \\\"route-monitor-opera-98lwvxopk1c4mcyxq48sgl92b67gbgycw0kkd4rdsr5\\\" in namespace \\\"\\\": Unauthorized\"\ntime=2026-03-16T20:05:58.181Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy\ntime=2026-03-16T20:05:58.181Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy\ntime=2026-03-16T20:05:58.291Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy err=\"failed to get status for release route-monitor-operator: release: not found\"\ntime=2026-03-16T20:05:58.291Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy\ntime=2026-03-16T20:05:58.291Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: unable to continue with install: could not get information about the resource ClusterRole \\\"route-monitor-opera-98lwvxopk1c4mcyxq48sgl92b67gbgycw0kkd4rdsr5\\\" in namespace \\\"\\\": Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1d9a3a5e025bac49a20fb6a589a05c4818d7ea04b6ac1a928d7474027746d37f","failure_pattern":"unable to continue with install: could not get information about the resource ClusterRole \"route-monitor-opera-98lwvxopk1c4mcyxq48sgl92b67gbgycw0kkd4rdsr5\" in namespace \"\": Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: unable to continue with install: could not get information about the resource ClusterRole \\\"route-monitor-opera-98lwvxopk1c4mcyxq48sgl92b67gbgycw0kkd4rdsr5\\\" in namespace \\\"\\\": Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1d9a3a5e025bac49a20fb6a589a05c4818d7ea04b6ac1a928d7474027746d37f"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033624356210348032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4219,"pr_state":"closed","pr_sha":"ad2d12b8fdba06f94d34e2abe00766c63912193c","final_merged_sha":"8550ece78eb685f5f0357a6937bf6b6e9f457543","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T19:20:04Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"78fed701ccb20a8d5a638a2c6888c7e94e10178d0665911cd6876ee7dcef453c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033624356210348032","occurred_at":"2026-03-16T19:20:04Z","failed_at":"provision","signature_id":"5a16efce6856cb7ec3c289414cb5c6df861b61434d949293d8a83736a8fb478f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-16T19:59:43.936Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Shell\\n Command: make deploy\\n\"\ntime=2026-03-16T20:00:56.397Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Shell Step, failed to execute shell command: make[2]: Entering directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\nkubectl create namespace clusters-service --dry-run=client -o json | kubectl apply -f - \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_CLIENT_ID=a296ceb2-a43e-4725-9169-304a0c3e17b6 \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_TENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nkubectl label namespace clusters-service \\\"istio.io/rev=prod-stable\\\" --overwrite=true \u0026\u0026 \\\\\\nAZURE_CS_MI_CLIENT_ID=2e022157-7935-49d1-9846-1981212496c7 \u0026\u0026 \\\\\\nTENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nOIDC_BLOB_SERVICE_ENDPOINT=https://arohcpprowoidcj0348032.blob.core.windows.net/ \u0026\u0026 \\\\\\nOIDC_ISSUER_BASE_ENDPOINT=https://arohcpprowoidcj0348032.z1.web.core.windows.net/ \u0026\u0026 \\\\\\nDB_HOST=$(if [ \\\"false\\\" = \\\"true\\\" ]; then az postgres flexible-server show -g hcp-underlay-prow-j0348032 -n arohcp-prow-dbcs-j0348032 --query fullyQualifiedDomainName -o tsv; else echo \\\"ocm-cs-db\\\"; fi) \u0026\u0026 \\\\\\nOVERRIDES=$(if [ \\\"false\\\" = \\\"true\\\" ]; then echo \\\"azuredb.values.yaml\\\"; else echo \\\"containerdb.values.yaml\\\"; fi) \u0026\u0026 \\\\\\nZONE_RESOURCE_ID=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0348032/providers/Microsoft.Network/dnszones/j0348032.hcp.osadev.cloud \u0026\u0026 \\\\\\nCX_SECRETS_KV_URL=\\\"https://ah-prow-cx-j0348032-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\nCX_MI_KV_URL=\\\"https://ah-prow-mi-j0348032-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\n./hack/helm.sh cluster-service helm-charts/cluster-service clusters-service \\\\\\n -f helm-charts/cluster-service/${OVERRIDES} \\\\\\n --set serviceAccountName=clusters-service \\\\\\n --set environment=arohcpprow \\\\\\n --set azureCsMiClientId=${AZURE_CS_MI_CLIENT_ID} \\\\\\n --set oidcIssuerBlobServiceUrl=${OIDC_BLOB_SERVICE_ENDPOINT} \\\\\\n --set oidcIssuerBaseUrl=${OIDC_ISSUER_BASE_ENDPOINT} \\\\\\n --set tlsCertificatesIssuer=${TLS_CERTIFICATES_ISSUER} \\\\\\n --set tlsCertificatesEnabled=true \\\\\\n --set denyAssignments=disabled \\\\\\n --set tenantId=${TENANT_ID} \\\\\\n --set region=westus3 \\\\\\n --set serviceKeyvaultName=aro-hcp-dev-svc-kv \\\\\\n --set replicas=3 \\\\\\n --set imageRegistry=arohcpsvcdev.azurecr.io \\\\\\n --set imageRepository=app-sre/aro-hcp-clusters-service \\\\\\n --set imageDigest=sha256:6ec7ecb11515ea6f8be560c4c043b518ef5c5ccc2a0aaafb2a45ee79c25a9d80 \\\\\\n --set global.imageDigest=sha256:6ec7ecb11515ea6f8be560c4c043b518ef5c5ccc2a0aaafb2a45ee79c25a9d80 \\\\\\n --set azureFirstPartyApplicationClientId=b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 \\\\\\n --set batchProcessesDryRun=true \\\\\\n --set batchProcesses=\\\"\\\" \\\\\\n --set fpaCertName=firstPartyCert2 \\\\\\n --set ocpAcrResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev \\\\\\n --set ocpAcrUrl=arohcpocpdev.azurecr.io \\\\\\n --set shard.consumerName=\\\"hcp-underlay-j0348032-mgmt-1\\\" \\\\\\n --set shard.zoneResourceId=\\\"${ZONE_RESOURCE_ID}\\\" \\\\\\n --set shard.cxSecretsKeyVaultUrl=\\\"${CX_SECRETS_KV_URL}\\\" \\\\\\n --set shard.cxMiKeyVaultUrl=\\\"${CX_MI_KV_URL}\\\" \\\\\\n --set shard.cxSecretsKeyVaultMiClientId=\\\"95e11a6d-913e-4adb-9890-9e0ebd50bfa2\\\" \\\\\\n --set shard.maestroRestUrl=\\\"http://maestro.maestro.svc.cluster.local:8000\\\" \\\\\\n --set shard.maestroGrpUrl=\\\"maestro-grpc.maestro.svc.cluster.local:8090\\\" \\\\\\n --set shard.Id=$(python3 -c \\\"import uuid; print(uuid.uuid5(uuid.UUID('916f9976-e1c0-4afd-b84c-5d5c94fbeaed'), 'prow-j0348032-mgmt-1'))\\\") \\\\\\n --set shard.aksManagementClusterResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0348032-mgmt-1/providers/Microsoft.ContainerService/managedClusters/prow-j0348032-mgmt-1 \\\\\\n --set databaseHost=${DB_HOST} \\\\\\n --set azureMiMockServicePrincipalPrincipalId=d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 \\\\\\n --set azureMiMockServicePrincipalClientId=e8723db7-9b9e-46a4-9f7d-64d75c3534f0 \\\\\\n --set azureMiMockServicePrincipalCertName=msiMockCert2 \\\\\\n --set azureArmHelperIdentityCertName=armHelperCert2 \\\\\\n --set azureArmHelperIdentityClientId=3331e670-0804-48e8-a086-6241671ddc93 \\\\\\n --set azureArmHelperMockFpaPrincipalId=47f69502-0065-4d9a-b19b-d403e183d2f4 \\\\\\n --set azureCloudEnvironmentName=AzurePublicCloud \\\\\\n --set pullBinding.workloadIdentityClientId=\\\"${IMAGE_PULLER_MI_CLIENT_ID}\\\" \\\\\\n --set pullBinding.workloadIdentityTenantId=\\\"${IMAGE_PULLER_MI_TENANT_ID}\\\" \\\\\\n --set pullBinding.registry=arohcpsvcdev.azurecr.io \\\\\\n --set pullBinding.scope=repository:app-sre/aro-hcp-clusters-service:pull \\\\\\n --set managedIdentitiesDataPlaneAudienceResource=https://dummy.org \\\\\\n --set tracing.address=http://ingest.observability:4318 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxSurge=1 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxUnavailable=0 \\\\\\n --set provisionShardClusterLimit=100 \\\\\\n --set ocpVersions.defaultVersion.version=4.20.8 \\\\\\n --set ocpVersions.channelGroups.stable.minVersion=4.19.0 \\\\\\n --set ocpVersions.channelGroups.stable.maxVersion= \\\\\\n --set ocpVersions.channelGroups.candidate.minVersion=4.19.7 \\\\\\n --set ocpVersions.channelGroups.candidate.maxVersion= \\\\\\n --set ocpVersions.channelGroups.nightly.minVersion=4.19.0-0.nightly-20200101 \\\\\\n --set ocpVersions.channelGroups.nightly.maxVersion= \\\\\\n --set azureOperatorsMI.roleSetName=dev \\\\\\n --set deployment.zoneCount=4\\nnamespace/clusters-service created\\nnamespace/clusters-service labeled\\nRun Helm upgrade with release name cluster-service in namespace clusters-service\\nRelease \\\"cluster-service\\\" does not exist. Installing it now.\\nError: Unauthorized\\nmake[2]: *** [Makefile:32: deploy] Error 1\\nmake[2]: Leaving directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\n exit status 2\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40","failure_pattern":"make[2]: *** [Makefile:32: deploy] Error 1","search_query":"make[2]: *** [Makefile:32: deploy] Error 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033623844563980288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4467,"pr_state":"closed","pr_sha":"4834b3b749099f5b743a5d85207f30a5c96a61d4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T19:18:02Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"81c4b24825063eb2e3b43fc8ea975359625bafdbdfda3c21dec61532112e0a70","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033623844563980288","occurred_at":"2026-03-16T19:18:02Z","failed_at":"provision","signature_id":"5a16efce6856cb7ec3c289414cb5c6df861b61434d949293d8a83736a8fb478f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-16T19:59:44.540Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Shell\\n Command: make deploy\\n\"\ntime=2026-03-16T20:01:14.547Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Shell Step, failed to execute shell command: make[2]: Entering directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\nkubectl create namespace clusters-service --dry-run=client -o json | kubectl apply -f - \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_CLIENT_ID=d47d6e1b-67d5-43b3-8321-cc3d6e757e42 \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_TENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nkubectl label namespace clusters-service \\\"istio.io/rev=prod-stable\\\" --overwrite=true \u0026\u0026 \\\\\\nAZURE_CS_MI_CLIENT_ID=a4766fe2-bcce-4945-9d82-6beb905a1e30 \u0026\u0026 \\\\\\nTENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nOIDC_BLOB_SERVICE_ENDPOINT=https://arohcpprowoidcj3980288.blob.core.windows.net/ \u0026\u0026 \\\\\\nOIDC_ISSUER_BASE_ENDPOINT=https://arohcpprowoidcj3980288.z1.web.core.windows.net/ \u0026\u0026 \\\\\\nDB_HOST=$(if [ \\\"false\\\" = \\\"true\\\" ]; then az postgres flexible-server show -g hcp-underlay-prow-j3980288 -n arohcp-prow-dbcs-j3980288 --query fullyQualifiedDomainName -o tsv; else echo \\\"ocm-cs-db\\\"; fi) \u0026\u0026 \\\\\\nOVERRIDES=$(if [ \\\"false\\\" = \\\"true\\\" ]; then echo \\\"azuredb.values.yaml\\\"; else echo \\\"containerdb.values.yaml\\\"; fi) \u0026\u0026 \\\\\\nZONE_RESOURCE_ID=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j3980288/providers/Microsoft.Network/dnszones/j3980288.hcp.osadev.cloud \u0026\u0026 \\\\\\nCX_SECRETS_KV_URL=\\\"https://ah-prow-cx-j3980288-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\nCX_MI_KV_URL=\\\"https://ah-prow-mi-j3980288-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\n./hack/helm.sh cluster-service helm-charts/cluster-service clusters-service \\\\\\n -f helm-charts/cluster-service/${OVERRIDES} \\\\\\n --set serviceAccountName=clusters-service \\\\\\n --set environment=arohcpprow \\\\\\n --set azureCsMiClientId=${AZURE_CS_MI_CLIENT_ID} \\\\\\n --set oidcIssuerBlobServiceUrl=${OIDC_BLOB_SERVICE_ENDPOINT} \\\\\\n --set oidcIssuerBaseUrl=${OIDC_ISSUER_BASE_ENDPOINT} \\\\\\n --set tlsCertificatesIssuer=${TLS_CERTIFICATES_ISSUER} \\\\\\n --set tlsCertificatesEnabled=true \\\\\\n --set denyAssignments=disabled \\\\\\n --set tenantId=${TENANT_ID} \\\\\\n --set region=westus3 \\\\\\n --set serviceKeyvaultName=aro-hcp-dev-svc-kv \\\\\\n --set replicas=3 \\\\\\n --set imageRegistry=arohcpsvcdev.azurecr.io \\\\\\n --set imageRepository=app-sre/aro-hcp-clusters-service \\\\\\n --set imageDigest=sha256:6ec7ecb11515ea6f8be560c4c043b518ef5c5ccc2a0aaafb2a45ee79c25a9d80 \\\\\\n --set global.imageDigest=sha256:6ec7ecb11515ea6f8be560c4c043b518ef5c5ccc2a0aaafb2a45ee79c25a9d80 \\\\\\n --set azureFirstPartyApplicationClientId=b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 \\\\\\n --set batchProcessesDryRun=true \\\\\\n --set batchProcesses=\\\"\\\" \\\\\\n --set fpaCertName=firstPartyCert2 \\\\\\n --set ocpAcrResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev \\\\\\n --set ocpAcrUrl=arohcpocpdev.azurecr.io \\\\\\n --set shard.consumerName=\\\"hcp-underlay-j3980288-mgmt-1\\\" \\\\\\n --set shard.zoneResourceId=\\\"${ZONE_RESOURCE_ID}\\\" \\\\\\n --set shard.cxSecretsKeyVaultUrl=\\\"${CX_SECRETS_KV_URL}\\\" \\\\\\n --set shard.cxMiKeyVaultUrl=\\\"${CX_MI_KV_URL}\\\" \\\\\\n --set shard.cxSecretsKeyVaultMiClientId=\\\"7fac78f7-d715-4063-b78f-ca3c8ffeb1ef\\\" \\\\\\n --set shard.maestroRestUrl=\\\"http://maestro.maestro.svc.cluster.local:8000\\\" \\\\\\n --set shard.maestroGrpUrl=\\\"maestro-grpc.maestro.svc.cluster.local:8090\\\" \\\\\\n --set shard.Id=$(python3 -c \\\"import uuid; print(uuid.uuid5(uuid.UUID('916f9976-e1c0-4afd-b84c-5d5c94fbeaed'), 'prow-j3980288-mgmt-1'))\\\") \\\\\\n --set shard.aksManagementClusterResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j3980288-mgmt-1/providers/Microsoft.ContainerService/managedClusters/prow-j3980288-mgmt-1 \\\\\\n --set databaseHost=${DB_HOST} \\\\\\n --set azureMiMockServicePrincipalPrincipalId=d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 \\\\\\n --set azureMiMockServicePrincipalClientId=e8723db7-9b9e-46a4-9f7d-64d75c3534f0 \\\\\\n --set azureMiMockServicePrincipalCertName=msiMockCert2 \\\\\\n --set azureArmHelperIdentityCertName=armHelperCert2 \\\\\\n --set azureArmHelperIdentityClientId=3331e670-0804-48e8-a086-6241671ddc93 \\\\\\n --set azureArmHelperMockFpaPrincipalId=47f69502-0065-4d9a-b19b-d403e183d2f4 \\\\\\n --set azureCloudEnvironmentName=AzurePublicCloud \\\\\\n --set pullBinding.workloadIdentityClientId=\\\"${IMAGE_PULLER_MI_CLIENT_ID}\\\" \\\\\\n --set pullBinding.workloadIdentityTenantId=\\\"${IMAGE_PULLER_MI_TENANT_ID}\\\" \\\\\\n --set pullBinding.registry=arohcpsvcdev.azurecr.io \\\\\\n --set pullBinding.scope=repository:app-sre/aro-hcp-clusters-service:pull \\\\\\n --set managedIdentitiesDataPlaneAudienceResource=https://dummy.org \\\\\\n --set tracing.address=http://ingest.observability:4318 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxSurge=1 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxUnavailable=0 \\\\\\n --set provisionShardClusterLimit=100 \\\\\\n --set ocpVersions.defaultVersion.version=4.20.8 \\\\\\n --set ocpVersions.channelGroups.stable.minVersion=4.19.0 \\\\\\n --set ocpVersions.channelGroups.stable.maxVersion= \\\\\\n --set ocpVersions.channelGroups.candidate.minVersion=4.19.7 \\\\\\n --set ocpVersions.channelGroups.candidate.maxVersion= \\\\\\n --set ocpVersions.channelGroups.nightly.minVersion=4.19.0-0.nightly-20200101 \\\\\\n --set ocpVersions.channelGroups.nightly.maxVersion= \\\\\\n --set azureOperatorsMI.roleSetName=dev \\\\\\n --set deployment.zoneCount=4\\nnamespace/clusters-service created\\nnamespace/clusters-service labeled\\nRun Helm upgrade with release name cluster-service in namespace clusters-service\\nRelease \\\"cluster-service\\\" does not exist. Installing it now.\\nError: Unauthorized\\nmake[2]: *** [Makefile:32: deploy] Error 1\\nmake[2]: Leaving directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\n exit status 2\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40","failure_pattern":"make[2]: *** [Makefile:32: deploy] Error 1","search_query":"make[2]: *** [Makefile:32: deploy] Error 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4449/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033622865810558976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4449,"pr_state":"closed","pr_sha":"205984e6ef0d1b76b1e624468408854ec2a65a8c","final_merged_sha":"205984e6ef0d1b76b1e624468408854ec2a65a8c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T19:14:08Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"6762a1f918fe3970877d05e9568d7d49f02e7a6625a12b16c384e1a6296c529e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4449/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033622865810558976","occurred_at":"2026-03-16T19:14:08Z","failed_at":"provision","signature_id":"a5b5f71a061e60f11c9c511d431eae042d41346983d1442cca50d855704dabbe","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/prometheus","test_suite":"step graph","failure_text":"time=2026-03-16T19:29:50.886Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-03-16T19:29:51.578Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[customResourceState:map[config:map[spec:map[resources:[map[groupVersionKind:map[group:hypershift.openshift.io kind:HostedCluster version:v1beta1] labelsFromPath:map[_id:[spec clusterID] api_url:[spec services [service=APIServer] servicePublishingStrategy route hostname] name:[metadata name] namespace:[metadata namespace]] metricNamePrefix:hostedClusterAPI metrics:[map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=KubeAPIServerAvailable] status]] type:StateSet] help:KubeAPIServer availability condition name:kubeapiserver_available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Available] status]] type:StateSet] help:Cluster availability condition name:available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Progressing] status]] type:StateSet] help:Cluster progressing condition name:progressing]]]]]] create:true enabled:true key:config.yaml name:kube-state-metric-custom-resource-config] image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c] rbac:map[extraRules:[map[apiGroups:[hypershift.openshift.io] resources:[hostedclusters] verbs:[list watch]]]]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:d7e8257f8d8bce64b6df55f81fba92011a6a77269b3350f8b997b152af348dba]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:a76595ab-de64-4e87-a475-d43fd793c26e]] prometheusSpec:map[externalLabels:map[cluster:prow-j0558976-mgmt-1] hcpRemoteWriteUrl:https://msprom-westus3-prow-j0558976-mgmt-1-xnqn.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-855a3699732143c49a482e7b776f3104/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:dca4b35f73046dc4803c0160c2e6be5a1736a716b26ef35590864d36dfdf950a] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j0558976-mgmt-1-xnqn.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-9011cef126304e52a16fcc6c1686f3bb/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-16T19:29:51.578Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T19:29:51.916Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus namespace=prometheus\ntime=2026-03-16T19:29:52.287Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus dryRun=false\ntime=2026-03-16T19:29:52.287Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T19:29:52.463Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T19:30:50.509Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"failed post-install: Unauthorized\"\ntime=2026-03-16T19:30:50.509Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T19:30:50.509Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T19:30:57.439Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus release=arohcp-monitor namespace=prometheus status=failed description=\"Release \\\"arohcp-monitor\\\" failed: failed post-install: Unauthorized\" values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[customResourceState:map[config:map[spec:map[resources:[map[groupVersionKind:map[group:hypershift.openshift.io kind:HostedCluster version:v1beta1] labelsFromPath:map[_id:[spec clusterID] api_url:[spec services [service=APIServer] servicePublishingStrategy route hostname] name:[metadata name] namespace:[metadata namespace]] metricNamePrefix:hostedClusterAPI metrics:[map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=KubeAPIServerAvailable] status]] type:StateSet] help:KubeAPIServer availability condition name:kubeapiserver_available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Available] status]] type:StateSet] help:Cluster availability condition name:available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Progressing] status]] type:StateSet] help:Cluster progressing condition name:progressing]]]]]] create:true enabled:true key:config.yaml name:kube-state-metric-custom-resource-config] image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c] rbac:map[extraRules:[map[apiGroups:[hypershift.openshift.io] resources:[hostedclusters] verbs:[list watch]]]]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:d7e8257f8d8bce64b6df55f81fba92011a6a77269b3350f8b997b152af348dba]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:a76595ab-de64-4e87-a475-d43fd793c26e]] prometheusSpec:map[externalLabels:map[cluster:prow-j0558976-mgmt-1] hcpRemoteWriteUrl:https://msprom-westus3-prow-j0558976-mgmt-1-xnqn.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-855a3699732143c49a482e7b776f3104/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:dca4b35f73046dc4803c0160c2e6be5a1736a716b26ef35590864d36dfdf950a] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j0558976-mgmt-1-xnqn.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-9011cef126304e52a16fcc6c1686f3bb/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-16T19:30:57.441Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/6yWz27TQBDG736KlU+NlI3SVK1oUA9QOKGiqiCQijisdwdnG+/OanbcEKkPjxw3jVvqYOwcPZ75ebx/5vuSAlgQRCxJQxQXwihWrLICjjC7A82frDfzyGR9PhZ16LNy8BSCe/CbSAxKb8OjH4kQQqRfgO6thndaY+k5HYtUES50kA69ZSS5LDOQkRWDdMBkdaySAqEDXkAZ03EbaJcjMQApRvr/0tcrLosyMtANFvDeemN93rnz/YiWnrdFHyAUuHYwaKWeQzqv0jdVWKPY+vw7ZAvE5SX6XzYvSbFF/wKljLMxPsa3hGs0V3WvXdb4uvX9/uybsoCX3RRA7JRXOdCEygL6QPTmdyVBgcoA9UHk4IFU0buF5ZtY1040elbWA0kHDmkttdILOCyS4oF7jCsVDktcIS2tz2UEltmaD7ao23E3CBfQSFx5aLlP+ynVXW48ezQgCTRSNSn6H6COI+KfEPLAEKUKYSDhSViGYSIjqbzXfjUg68jgpAo2At333rdntB6MzV7D74DEA8bVM0rP+t4f7ywqHSktKryZyFf1UFFObc90NQ4qndrObFdn1If/cVf+EuEhAt5BuVuzm0mPRmS4C+pjfxrSPOzjr2p8Ww/Jz7fJ7tJ8rKxiTB7EagEEgq2DyMoFkQGvALw4MoqhCh+ls+nsTE5P5PHZ1+Pz+ex8fjq7TUdiMhFtOSfT+en0Nh2Nkgdxh9aLpfXmwnoP1DC46MXO1zYN7UsnmzyIQFi93TU6FgFNndwNMhYF5mPhIEaVV0gkAySydePnDUT9JwAA///iuuQVhgsAAA==\"\ntime=2026-03-16T19:30:57.442Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T19:30:57.443Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: failed post-install: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82e6580bc32e56aa7378422fd2960d02a2eebbb9fbfbc08994384c40137bbab6","failure_pattern":"failed post-install: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed post-install: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["82e6580bc32e56aa7378422fd2960d02a2eebbb9fbfbc08994384c40137bbab6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033621699479474176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4045,"pr_state":"open","pr_sha":"3207545fca598a0bbc926207da9f3ebd12d86873","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T19:09:30Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"d79d1320185dc89f49ae26635c629286b9f62f3831d74f70e886369334e99566","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033621699479474176","occurred_at":"2026-03-16T19:09:30Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4095/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033621611264872448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4095,"pr_state":"open","pr_sha":"8f711fab81a8d9300197820c371560b67cc2acd0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T19:09:09Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"3e11ef6e55b1a824016563965240946c05f93bb4acc75a6fa45d3bdee4ab44db","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4095/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033621611264872448","occurred_at":"2026-03-16T19:09:09Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033621584618459136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4393,"pr_state":"open","pr_sha":"dcec1a0f0e9fc5822919f091837faed741634449","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T19:09:03Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"fe955d8c17a45ae9f666ff426442c11cd72cbbdeb81eb86f8ec5e7cf5302edd9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033621584618459136","occurred_at":"2026-03-16T19:09:03Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033621557934297088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4457,"pr_state":"open","pr_sha":"c6d34c6bc54bf82f40b7a2b9e91f43b977c11e31","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T19:08:56Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3baffc5d5e6e1926ce6eead073d8d0917dbd8937d21b4d5f03a38ff9e718971c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033621557934297088","occurred_at":"2026-03-16T19:08:56Z","failed_at":"provision","signature_id":"5a16efce6856cb7ec3c289414cb5c6df861b61434d949293d8a83736a8fb478f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-16T19:45:32.200Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Shell\\n Command: make deploy\\n\"\ntime=2026-03-16T19:45:58.890Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Shell Step, failed to execute shell command: make[2]: Entering directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\nkubectl create namespace clusters-service --dry-run=client -o json | kubectl apply -f - \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_CLIENT_ID=6ad615cf-74ad-45b9-98fd-af2a5733844d \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_TENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nkubectl label namespace clusters-service \\\"istio.io/rev=prod-stable\\\" --overwrite=true \u0026\u0026 \\\\\\nAZURE_CS_MI_CLIENT_ID=f52e7edf-bae6-411f-bce8-6a372bd4b676 \u0026\u0026 \\\\\\nTENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nOIDC_BLOB_SERVICE_ENDPOINT=https://arohcpprowoidcj4297088.blob.core.windows.net/ \u0026\u0026 \\\\\\nOIDC_ISSUER_BASE_ENDPOINT=https://arohcpprowoidcj4297088.z1.web.core.windows.net/ \u0026\u0026 \\\\\\nDB_HOST=$(if [ \\\"false\\\" = \\\"true\\\" ]; then az postgres flexible-server show -g hcp-underlay-prow-j4297088 -n arohcp-prow-dbcs-j4297088 --query fullyQualifiedDomainName -o tsv; else echo \\\"ocm-cs-db\\\"; fi) \u0026\u0026 \\\\\\nOVERRIDES=$(if [ \\\"false\\\" = \\\"true\\\" ]; then echo \\\"azuredb.values.yaml\\\"; else echo \\\"containerdb.values.yaml\\\"; fi) \u0026\u0026 \\\\\\nZONE_RESOURCE_ID=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4297088/providers/Microsoft.Network/dnszones/j4297088.hcp.osadev.cloud \u0026\u0026 \\\\\\nCX_SECRETS_KV_URL=\\\"https://ah-prow-cx-j4297088-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\nCX_MI_KV_URL=\\\"https://ah-prow-mi-j4297088-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\n./hack/helm.sh cluster-service helm-charts/cluster-service clusters-service \\\\\\n -f helm-charts/cluster-service/${OVERRIDES} \\\\\\n --set serviceAccountName=clusters-service \\\\\\n --set environment=arohcpprow \\\\\\n --set azureCsMiClientId=${AZURE_CS_MI_CLIENT_ID} \\\\\\n --set oidcIssuerBlobServiceUrl=${OIDC_BLOB_SERVICE_ENDPOINT} \\\\\\n --set oidcIssuerBaseUrl=${OIDC_ISSUER_BASE_ENDPOINT} \\\\\\n --set tlsCertificatesIssuer=${TLS_CERTIFICATES_ISSUER} \\\\\\n --set tlsCertificatesEnabled=true \\\\\\n --set denyAssignments=disabled \\\\\\n --set tenantId=${TENANT_ID} \\\\\\n --set region=westus3 \\\\\\n --set serviceKeyvaultName=aro-hcp-dev-svc-kv \\\\\\n --set replicas=3 \\\\\\n --set imageRegistry=arohcpsvcdev.azurecr.io \\\\\\n --set imageRepository=app-sre/aro-hcp-clusters-service \\\\\\n --set imageDigest=sha256:6ec7ecb11515ea6f8be560c4c043b518ef5c5ccc2a0aaafb2a45ee79c25a9d80 \\\\\\n --set global.imageDigest=sha256:6ec7ecb11515ea6f8be560c4c043b518ef5c5ccc2a0aaafb2a45ee79c25a9d80 \\\\\\n --set azureFirstPartyApplicationClientId=b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 \\\\\\n --set batchProcessesDryRun=true \\\\\\n --set batchProcesses=\\\"\\\" \\\\\\n --set fpaCertName=firstPartyCert2 \\\\\\n --set ocpAcrResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev \\\\\\n --set ocpAcrUrl=arohcpocpdev.azurecr.io \\\\\\n --set shard.consumerName=\\\"hcp-underlay-j4297088-mgmt-1\\\" \\\\\\n --set shard.zoneResourceId=\\\"${ZONE_RESOURCE_ID}\\\" \\\\\\n --set shard.cxSecretsKeyVaultUrl=\\\"${CX_SECRETS_KV_URL}\\\" \\\\\\n --set shard.cxMiKeyVaultUrl=\\\"${CX_MI_KV_URL}\\\" \\\\\\n --set shard.cxSecretsKeyVaultMiClientId=\\\"b6de3e02-8f2f-4bf8-9489-ac97663eecaa\\\" \\\\\\n --set shard.maestroRestUrl=\\\"http://maestro.maestro.svc.cluster.local:8000\\\" \\\\\\n --set shard.maestroGrpUrl=\\\"maestro-grpc.maestro.svc.cluster.local:8090\\\" \\\\\\n --set shard.Id=$(python3 -c \\\"import uuid; print(uuid.uuid5(uuid.UUID('916f9976-e1c0-4afd-b84c-5d5c94fbeaed'), 'prow-j4297088-mgmt-1'))\\\") \\\\\\n --set shard.aksManagementClusterResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4297088-mgmt-1/providers/Microsoft.ContainerService/managedClusters/prow-j4297088-mgmt-1 \\\\\\n --set databaseHost=${DB_HOST} \\\\\\n --set azureMiMockServicePrincipalPrincipalId=d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 \\\\\\n --set azureMiMockServicePrincipalClientId=e8723db7-9b9e-46a4-9f7d-64d75c3534f0 \\\\\\n --set azureMiMockServicePrincipalCertName=msiMockCert2 \\\\\\n --set azureArmHelperIdentityCertName=armHelperCert2 \\\\\\n --set azureArmHelperIdentityClientId=3331e670-0804-48e8-a086-6241671ddc93 \\\\\\n --set azureArmHelperMockFpaPrincipalId=47f69502-0065-4d9a-b19b-d403e183d2f4 \\\\\\n --set azureCloudEnvironmentName=AzurePublicCloud \\\\\\n --set pullBinding.workloadIdentityClientId=\\\"${IMAGE_PULLER_MI_CLIENT_ID}\\\" \\\\\\n --set pullBinding.workloadIdentityTenantId=\\\"${IMAGE_PULLER_MI_TENANT_ID}\\\" \\\\\\n --set pullBinding.registry=arohcpsvcdev.azurecr.io \\\\\\n --set pullBinding.scope=repository:app-sre/aro-hcp-clusters-service:pull \\\\\\n --set managedIdentitiesDataPlaneAudienceResource=https://dummy.org \\\\\\n --set tracing.address=http://ingest.observability:4318 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxSurge=1 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxUnavailable=0 \\\\\\n --set provisionShardClusterLimit=100 \\\\\\n --set ocpVersions.defaultVersion.version=4.20.8 \\\\\\n --set ocpVersions.channelGroups.stable.minVersion=4.19.0 \\\\\\n --set ocpVersions.channelGroups.stable.maxVersion= \\\\\\n --set ocpVersions.channelGroups.candidate.minVersion=4.19.7 \\\\\\n --set ocpVersions.channelGroups.candidate.maxVersion= \\\\\\n --set ocpVersions.channelGroups.nightly.minVersion=4.19.0-0.nightly-20200101 \\\\\\n --set ocpVersions.channelGroups.nightly.maxVersion= \\\\\\n --set azureOperatorsMI.roleSetName=dev \\\\\\n --set deployment.zoneCount=4\\nnamespace/clusters-service created\\nnamespace/clusters-service labeled\\nRun Helm upgrade with release name cluster-service in namespace clusters-service\\nRelease \\\"cluster-service\\\" does not exist. Installing it now.\\nError: Unauthorized\\nmake[2]: *** [Makefile:32: deploy] Error 1\\nmake[2]: Leaving directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\n exit status 2\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40","failure_pattern":"make[2]: *** [Makefile:32: deploy] Error 1","search_query":"make[2]: *** [Makefile:32: deploy] Error 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4466/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033619460677439488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4466,"pr_state":"closed","pr_sha":"8cb4bd5f1213fd23ab6fcccc158048912ff9661e","final_merged_sha":"8cb4bd5f1213fd23ab6fcccc158048912ff9661e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T19:00:36Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"feacffc75649f9d0fe0578daa66c69424532b54e95db9a124131fcedcf7e3844","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4466/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033619460677439488","occurred_at":"2026-03-16T19:00:36Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033619047349751808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4454,"pr_state":"closed","pr_sha":"38f866ad42f9613c0978385116cd5b0c6df796c1","final_merged_sha":"38f866ad42f9613c0978385116cd5b0c6df796c1","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-16T18:58:57Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4455/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033619009596821504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4455,"pr_state":"closed","pr_sha":"7b9532af16d82b9a9786a1179527a9bf0cc955bb","final_merged_sha":"7b9532af16d82b9a9786a1179527a9bf0cc955bb","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T18:58:42Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"028a81dfbdf31c7563ddb7ccfaa4f16bd59358e47edd46ec4259fba0423fe914","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4455/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033619009596821504","occurred_at":"2026-03-16T18:58:42Z","failed_at":"provision","signature_id":"edc1ac697f2382e629e25b931b72d04a0489f13c9ef9965424c7a71d36aab01d","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Agent/management/deploy","test_suite":"step graph","failure_text":"time=2026-03-16T19:20:03.896Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-16T19:20:04.528Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy values=\"map[azure:map[clientId:4044d85c-56be-4b90-b184-215a1e0a5de8 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] broker:map[host:arohcp-prow-maestro-j6821504.westus3-1.ts.eventgrid.azure.net port:8883] consumerName:hcp-underlay-j6821504-mgmt-1 credsKeyVault:map[name:ah-prow-mg-j6821504-1 secret:hcp-underlay-j6821504-mgmt-1] deployment:map[zoneCount:4] glog_v:2 image:map[digest:sha256:391b002a10ffd994fa4f47e4d1096bac91e512769c0073df782f3cd8d8fe43e7 registry:arohcpsvcdev.azurecr.io repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro] installAppliedManifestWorkCRD:false replicas:3 serviceAccountName:maestro sideCar:map[image:map[digest:sha256:331aaa0d553e8eec9b69261f1dc6fd74eadc1daec5907287dc5e9bbc3a7e2bc0 registry:mcr.microsoft.com repository:azurelinux/base/nginx]]]\"\ntime=2026-03-16T19:20:04.528Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy\ntime=2026-03-16T19:20:05.320Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy namespace=maestro\ntime=2026-03-16T19:20:05.606Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy dryRun=false\ntime=2026-03-16T19:20:05.606Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy\ntime=2026-03-16T19:20:06.157Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy\ntime=2026-03-16T19:20:08.352Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy err=\"unable to continue with install: could not get information about the resource ServiceAccount \\\"metrics-proxy-sa\\\" in namespace \\\"maestro\\\": Unauthorized\"\ntime=2026-03-16T19:20:08.352Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy\ntime=2026-03-16T19:20:08.352Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy\ntime=2026-03-16T19:20:08.461Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy err=\"failed to get status for release maestro-agent: release: not found\"\ntime=2026-03-16T19:20:08.461Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy\ntime=2026-03-16T19:20:08.462Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: unable to continue with install: could not get information about the resource ServiceAccount \\\"metrics-proxy-sa\\\" in namespace \\\"maestro\\\": Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"defa6ad4995a2b7ba7ef8750880def2f331b60498e4a50ae9987ac13f6614711","failure_pattern":"unable to continue with install: could not get information about the resource ServiceAccount \"metrics-proxy-sa\" in namespace \"\u003cnamespace\u003e\": Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: unable to continue with install: could not get information about the resource ServiceAccount \\\"metrics-proxy-sa\\\" in namespace \\\"maestro\\\": Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["defa6ad4995a2b7ba7ef8750880def2f331b60498e4a50ae9987ac13f6614711"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4483/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033614201892638720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4483,"pr_state":"open","pr_sha":"c5356b74da140f99e74e5c7493d26cb8dff9fd87","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T18:39:43Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"7844d4dd4b657c14a052fd046a49512b59cce13263dd964595d610ec1796cc74","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4483/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033614201892638720","occurred_at":"2026-03-16T18:39:43Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033614080501092352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4468,"pr_state":"closed","pr_sha":"10f6db82fe4e37a551e5388b2dc0840ec0886921","final_merged_sha":"532823b3d4dd565d5b0521f892350ccadb788955","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T18:39:14Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"f22b794656e4e2f74738748d11c97fd99b087b493c87fae81d983ab6a38b76f1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033614080501092352","occurred_at":"2026-03-16T18:39:14Z","failed_at":"provision","signature_id":"da447673fb6341bf3e2299f015db2de75b1829546be65bcc65d6b29fcad4b415","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-16T19:20:18.975Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-16T19:20:19.506Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj1092352.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj1092352.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:2076d051-d5ae-4edc-b8d1-25ff9f8a0c39 databaseName:arohcpprow-rp-j1092352 databaseUrl:https://arohcpprow-rp-j1092352.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-r7hdm913/pipeline@sha256:9e3e8e42d97e5eec67d63504a6b1d5967d7ea2b8942b4a88ad9fc6730bc3d19c replicas:2 zoneCount:4] exitOnPanic:true fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] miMock:map[certName:msiMockCert2 clientId:e8723db7-9b9e-46a4-9f7d-64d75c3534f0 principalId:d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-r7hdm913/pipeline:pull workloadIdentityClientId:8ddc6831-4500-4dc8-93f7-0660f64237ec workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:2076d051-d5ae-4edc-b8d1-25ff9f8a0c39 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-16T19:20:19.506Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-16T19:20:20.138Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace aro-hcp: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"573e4e1f4c8c691cd1ac67801788343411e0420c5b789273f33bf0a63260366d","failure_pattern":"failed to apply namespace aro-hcp: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace aro-hcp: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["573e4e1f4c8c691cd1ac67801788343411e0420c5b789273f33bf0a63260366d"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033613906890461184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4305,"pr_state":"closed","pr_sha":"b35918607a6adb258ac5f345f985885c625ed92d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T18:38:32Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d089527479a0993d566d929e92afa346cfafe2c5a55ac6ca979dc7e329c082c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033613906890461184","occurred_at":"2026-03-16T18:38:32Z","failed_at":"provision","signature_id":"55f959212b276953d69124b2535dda8bcd5c609c88c398d3ca208722abf25ab9","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/management/deploy","test_suite":"step graph","failure_text":"time=2026-03-16T19:14:49.008Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-16T19:14:49.695Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy values=\"map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5 imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:490c3b795cb12bd9e5a3680bb7dabd7ea02799c30b14ed960611517c5d808e6b imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[cpu:50m memory:100Mi] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:\u003cnil\u003e]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]\"\ntime=2026-03-16T19:14:49.695Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy\ntime=2026-03-16T19:14:49.894Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy namespace=secret-sync-controller\ntime=2026-03-16T19:14:50.213Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy dryRun=false\ntime=2026-03-16T19:14:50.213Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy\ntime=2026-03-16T19:14:50.322Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy\ntime=2026-03-16T19:14:53.586Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy err=\"unable to continue with install: could not get information about the resource Deployment \\\"secrets-store-sync-controller-manager\\\" in namespace \\\"secret-sync-controller\\\": Unauthorized\"\ntime=2026-03-16T19:14:53.586Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy\ntime=2026-03-16T19:14:53.586Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy\ntime=2026-03-16T19:14:53.694Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy err=\"failed to get status for release secret-sync-controller: release: not found\"\ntime=2026-03-16T19:14:53.694Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy\ntime=2026-03-16T19:14:53.694Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: unable to continue with install: could not get information about the resource Deployment \\\"secrets-store-sync-controller-manager\\\" in namespace \\\"secret-sync-controller\\\": Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e9d93a3dcb4580e4bf0333d287c66ac2ab30479d288fd965e102bcf30328ab0d","failure_pattern":"unable to continue with install: could not get information about the resource Deployment \"secrets-store-sync-controller-manager\" in namespace \"\u003cnamespace\u003e\": Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: unable to continue with install: could not get information about the resource Deployment \\\"secrets-store-sync-controller-manager\\\" in namespace \\\"secret-sync-controller\\\": Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e9d93a3dcb4580e4bf0333d287c66ac2ab30479d288fd965e102bcf30328ab0d"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033609905897738240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4226,"pr_state":"closed","pr_sha":"3f588748fd29207330fe25d741a19c20e8d6fbb8","final_merged_sha":"138964dfc1d79a030eb9c07890259cb904e592a1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T18:22:33Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"458c86902b39e83c8792af88698bbdee88422424c601bf60b0d09060151f6446","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033609905897738240","occurred_at":"2026-03-16T18:22:33Z","failed_at":"provision","signature_id":"5a16efce6856cb7ec3c289414cb5c6df861b61434d949293d8a83736a8fb478f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-16T19:00:09.988Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Shell\\n Command: make deploy\\n\"\ntime=2026-03-16T19:00:24.322Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Shell Step, failed to execute shell command: make[2]: Entering directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\nkubectl create namespace clusters-service --dry-run=client -o json | kubectl apply -f - \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_CLIENT_ID=36a2b479-629f-4bdd-822e-834cb3abc063 \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_TENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nkubectl label namespace clusters-service \\\"istio.io/rev=prod-stable\\\" --overwrite=true \u0026\u0026 \\\\\\nAZURE_CS_MI_CLIENT_ID=fd72cb74-fdc0-40a7-b8ae-1d206ba66b31 \u0026\u0026 \\\\\\nTENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nOIDC_BLOB_SERVICE_ENDPOINT=https://arohcpprowoidcj7738240.blob.core.windows.net/ \u0026\u0026 \\\\\\nOIDC_ISSUER_BASE_ENDPOINT=https://arohcpprowoidcj7738240.z1.web.core.windows.net/ \u0026\u0026 \\\\\\nDB_HOST=$(if [ \\\"false\\\" = \\\"true\\\" ]; then az postgres flexible-server show -g hcp-underlay-prow-j7738240 -n arohcp-prow-dbcs-j7738240 --query fullyQualifiedDomainName -o tsv; else echo \\\"ocm-cs-db\\\"; fi) \u0026\u0026 \\\\\\nOVERRIDES=$(if [ \\\"false\\\" = \\\"true\\\" ]; then echo \\\"azuredb.values.yaml\\\"; else echo \\\"containerdb.values.yaml\\\"; fi) \u0026\u0026 \\\\\\nZONE_RESOURCE_ID=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7738240/providers/Microsoft.Network/dnszones/j7738240.hcp.osadev.cloud \u0026\u0026 \\\\\\nCX_SECRETS_KV_URL=\\\"https://ah-prow-cx-j7738240-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\nCX_MI_KV_URL=\\\"https://ah-prow-mi-j7738240-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\n./hack/helm.sh cluster-service helm-charts/cluster-service clusters-service \\\\\\n -f helm-charts/cluster-service/${OVERRIDES} \\\\\\n --set serviceAccountName=clusters-service \\\\\\n --set environment=arohcpprow \\\\\\n --set azureCsMiClientId=${AZURE_CS_MI_CLIENT_ID} \\\\\\n --set oidcIssuerBlobServiceUrl=${OIDC_BLOB_SERVICE_ENDPOINT} \\\\\\n --set oidcIssuerBaseUrl=${OIDC_ISSUER_BASE_ENDPOINT} \\\\\\n --set tlsCertificatesIssuer=${TLS_CERTIFICATES_ISSUER} \\\\\\n --set tlsCertificatesEnabled=true \\\\\\n --set denyAssignments=disabled \\\\\\n --set tenantId=${TENANT_ID} \\\\\\n --set region=westus3 \\\\\\n --set serviceKeyvaultName=aro-hcp-dev-svc-kv \\\\\\n --set replicas=3 \\\\\\n --set imageRegistry=arohcpsvcdev.azurecr.io \\\\\\n --set imageRepository=app-sre/aro-hcp-clusters-service \\\\\\n --set imageDigest=sha256:6ec7ecb11515ea6f8be560c4c043b518ef5c5ccc2a0aaafb2a45ee79c25a9d80 \\\\\\n --set global.imageDigest=sha256:6ec7ecb11515ea6f8be560c4c043b518ef5c5ccc2a0aaafb2a45ee79c25a9d80 \\\\\\n --set azureFirstPartyApplicationClientId=b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 \\\\\\n --set batchProcessesDryRun=true \\\\\\n --set batchProcesses=\\\"\\\" \\\\\\n --set fpaCertName=firstPartyCert2 \\\\\\n --set ocpAcrResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev \\\\\\n --set ocpAcrUrl=arohcpocpdev.azurecr.io \\\\\\n --set shard.consumerName=\\\"hcp-underlay-j7738240-mgmt-1\\\" \\\\\\n --set shard.zoneResourceId=\\\"${ZONE_RESOURCE_ID}\\\" \\\\\\n --set shard.cxSecretsKeyVaultUrl=\\\"${CX_SECRETS_KV_URL}\\\" \\\\\\n --set shard.cxMiKeyVaultUrl=\\\"${CX_MI_KV_URL}\\\" \\\\\\n --set shard.cxSecretsKeyVaultMiClientId=\\\"a9667be0-9e5b-481c-b8bf-3b7eb1890100\\\" \\\\\\n --set shard.maestroRestUrl=\\\"http://maestro.maestro.svc.cluster.local:8000\\\" \\\\\\n --set shard.maestroGrpUrl=\\\"maestro-grpc.maestro.svc.cluster.local:8090\\\" \\\\\\n --set shard.Id=$(python3 -c \\\"import uuid; print(uuid.uuid5(uuid.UUID('916f9976-e1c0-4afd-b84c-5d5c94fbeaed'), 'prow-j7738240-mgmt-1'))\\\") \\\\\\n --set shard.aksManagementClusterResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7738240-mgmt-1/providers/Microsoft.ContainerService/managedClusters/prow-j7738240-mgmt-1 \\\\\\n --set databaseHost=${DB_HOST} \\\\\\n --set azureMiMockServicePrincipalPrincipalId=d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 \\\\\\n --set azureMiMockServicePrincipalClientId=e8723db7-9b9e-46a4-9f7d-64d75c3534f0 \\\\\\n --set azureMiMockServicePrincipalCertName=msiMockCert2 \\\\\\n --set azureArmHelperIdentityCertName=armHelperCert2 \\\\\\n --set azureArmHelperIdentityClientId=3331e670-0804-48e8-a086-6241671ddc93 \\\\\\n --set azureArmHelperMockFpaPrincipalId=47f69502-0065-4d9a-b19b-d403e183d2f4 \\\\\\n --set azureCloudEnvironmentName=AzurePublicCloud \\\\\\n --set pullBinding.workloadIdentityClientId=\\\"${IMAGE_PULLER_MI_CLIENT_ID}\\\" \\\\\\n --set pullBinding.workloadIdentityTenantId=\\\"${IMAGE_PULLER_MI_TENANT_ID}\\\" \\\\\\n --set pullBinding.registry=arohcpsvcdev.azurecr.io \\\\\\n --set pullBinding.scope=repository:app-sre/aro-hcp-clusters-service:pull \\\\\\n --set managedIdentitiesDataPlaneAudienceResource=https://dummy.org \\\\\\n --set tracing.address=http://ingest.observability:4318 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxSurge=1 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxUnavailable=0 \\\\\\n --set provisionShardClusterLimit=100 \\\\\\n --set ocpVersions.defaultVersion.version=4.20.8 \\\\\\n --set ocpVersions.channelGroups.stable.minVersion=4.19.0 \\\\\\n --set ocpVersions.channelGroups.stable.maxVersion= \\\\\\n --set ocpVersions.channelGroups.candidate.minVersion=4.19.7 \\\\\\n --set ocpVersions.channelGroups.candidate.maxVersion= \\\\\\n --set ocpVersions.channelGroups.nightly.minVersion=4.19.0-0.nightly-20200101 \\\\\\n --set ocpVersions.channelGroups.nightly.maxVersion= \\\\\\n --set azureOperatorsMI.roleSetName=dev \\\\\\n --set deployment.zoneCount=4\\nError from server (InternalError): error when retrieving current configuration of:\\nResource: \\\"/v1, Resource=namespaces\\\", GroupVersionKind: \\\"/v1, Kind=Namespace\\\"\\nName: \\\"clusters-service\\\", Namespace: \\\"\\\"\\nfrom server for: \\\"STDIN\\\": an error on the server (\\\"Internal Server Error: \\\\\\\"/api/v1/namespaces/clusters-service\\\\\\\": Post \\\\\\\"https://guard.69b84f6a54541c000131187e.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": proxy error from localhost:9443 while dialing guard.69b84f6a54541c000131187e.svc.cluster.local:443, code 503: 503 Service Unavailable\\\") has prevented the request from succeeding (get namespaces clusters-service)\\nmake[2]: *** [Makefile:32: deploy] Error 1\\nmake[2]: Leaving directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\n exit status 2\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40","failure_pattern":"make[2]: *** [Makefile:32: deploy] Error 1","search_query":"make[2]: *** [Makefile:32: deploy] Error 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033606999689662464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4224,"pr_state":"closed","pr_sha":"89dbea9535e9da4e251791f532920158951c59f0","final_merged_sha":"0d2c310eee81cd65f33fd842930856c19dc160e3","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-16T18:11:05Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4455/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033601072316026880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4455,"pr_state":"closed","pr_sha":"7b9532af16d82b9a9786a1179527a9bf0cc955bb","final_merged_sha":"7b9532af16d82b9a9786a1179527a9bf0cc955bb","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T17:47:31Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"e8abfc9619a47c1ed07b7defa845b4c2ee6bbc44b672985d89237481288ac5a8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4455/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033601072316026880","occurred_at":"2026-03-16T17:47:31Z","failed_at":"provision","signature_id":"5a16efce6856cb7ec3c289414cb5c6df861b61434d949293d8a83736a8fb478f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-16T18:28:51.532Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Shell\\n Command: make deploy\\n\"\ntime=2026-03-16T18:29:53.534Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Shell Step, failed to execute shell command: make[2]: Entering directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\nkubectl create namespace clusters-service --dry-run=client -o json | kubectl apply -f - \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_CLIENT_ID=ebf9c2e8-f4e4-455e-93fc-72cb59ea132b \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_TENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nkubectl label namespace clusters-service \\\"istio.io/rev=prod-stable\\\" --overwrite=true \u0026\u0026 \\\\\\nAZURE_CS_MI_CLIENT_ID=adb25dcd-b00c-4348-8ad9-25c8e21b1f51 \u0026\u0026 \\\\\\nTENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nOIDC_BLOB_SERVICE_ENDPOINT=https://arohcpprowoidcj6026880.blob.core.windows.net/ \u0026\u0026 \\\\\\nOIDC_ISSUER_BASE_ENDPOINT=https://arohcpprowoidcj6026880.z1.web.core.windows.net/ \u0026\u0026 \\\\\\nDB_HOST=$(if [ \\\"false\\\" = \\\"true\\\" ]; then az postgres flexible-server show -g hcp-underlay-prow-j6026880 -n arohcp-prow-dbcs-j6026880 --query fullyQualifiedDomainName -o tsv; else echo \\\"ocm-cs-db\\\"; fi) \u0026\u0026 \\\\\\nOVERRIDES=$(if [ \\\"false\\\" = \\\"true\\\" ]; then echo \\\"azuredb.values.yaml\\\"; else echo \\\"containerdb.values.yaml\\\"; fi) \u0026\u0026 \\\\\\nZONE_RESOURCE_ID=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6026880/providers/Microsoft.Network/dnszones/j6026880.hcp.osadev.cloud \u0026\u0026 \\\\\\nCX_SECRETS_KV_URL=\\\"https://ah-prow-cx-j6026880-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\nCX_MI_KV_URL=\\\"https://ah-prow-mi-j6026880-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\n./hack/helm.sh cluster-service helm-charts/cluster-service clusters-service \\\\\\n -f helm-charts/cluster-service/${OVERRIDES} \\\\\\n --set serviceAccountName=clusters-service \\\\\\n --set environment=arohcpprow \\\\\\n --set azureCsMiClientId=${AZURE_CS_MI_CLIENT_ID} \\\\\\n --set oidcIssuerBlobServiceUrl=${OIDC_BLOB_SERVICE_ENDPOINT} \\\\\\n --set oidcIssuerBaseUrl=${OIDC_ISSUER_BASE_ENDPOINT} \\\\\\n --set tlsCertificatesIssuer=${TLS_CERTIFICATES_ISSUER} \\\\\\n --set tlsCertificatesEnabled=true \\\\\\n --set denyAssignments=disabled \\\\\\n --set tenantId=${TENANT_ID} \\\\\\n --set region=westus3 \\\\\\n --set serviceKeyvaultName=aro-hcp-dev-svc-kv \\\\\\n --set replicas=3 \\\\\\n --set imageRegistry=arohcpsvcdev.azurecr.io \\\\\\n --set imageRepository=app-sre/aro-hcp-clusters-service \\\\\\n --set imageDigest=sha256:6ec7ecb11515ea6f8be560c4c043b518ef5c5ccc2a0aaafb2a45ee79c25a9d80 \\\\\\n --set global.imageDigest=sha256:6ec7ecb11515ea6f8be560c4c043b518ef5c5ccc2a0aaafb2a45ee79c25a9d80 \\\\\\n --set azureFirstPartyApplicationClientId=b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 \\\\\\n --set batchProcessesDryRun=true \\\\\\n --set batchProcesses=\\\"\\\" \\\\\\n --set fpaCertName=firstPartyCert2 \\\\\\n --set ocpAcrResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev \\\\\\n --set ocpAcrUrl=arohcpocpdev.azurecr.io \\\\\\n --set shard.consumerName=\\\"hcp-underlay-j6026880-mgmt-1\\\" \\\\\\n --set shard.zoneResourceId=\\\"${ZONE_RESOURCE_ID}\\\" \\\\\\n --set shard.cxSecretsKeyVaultUrl=\\\"${CX_SECRETS_KV_URL}\\\" \\\\\\n --set shard.cxMiKeyVaultUrl=\\\"${CX_MI_KV_URL}\\\" \\\\\\n --set shard.cxSecretsKeyVaultMiClientId=\\\"b602f500-ed9f-4351-a0d5-0538cad403c3\\\" \\\\\\n --set shard.maestroRestUrl=\\\"http://maestro.maestro.svc.cluster.local:8000\\\" \\\\\\n --set shard.maestroGrpUrl=\\\"maestro-grpc.maestro.svc.cluster.local:8090\\\" \\\\\\n --set shard.Id=$(python3 -c \\\"import uuid; print(uuid.uuid5(uuid.UUID('916f9976-e1c0-4afd-b84c-5d5c94fbeaed'), 'prow-j6026880-mgmt-1'))\\\") \\\\\\n --set shard.aksManagementClusterResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6026880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/prow-j6026880-mgmt-1 \\\\\\n --set databaseHost=${DB_HOST} \\\\\\n --set azureMiMockServicePrincipalPrincipalId=d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 \\\\\\n --set azureMiMockServicePrincipalClientId=e8723db7-9b9e-46a4-9f7d-64d75c3534f0 \\\\\\n --set azureMiMockServicePrincipalCertName=msiMockCert2 \\\\\\n --set azureArmHelperIdentityCertName=armHelperCert2 \\\\\\n --set azureArmHelperIdentityClientId=3331e670-0804-48e8-a086-6241671ddc93 \\\\\\n --set azureArmHelperMockFpaPrincipalId=47f69502-0065-4d9a-b19b-d403e183d2f4 \\\\\\n --set azureCloudEnvironmentName=AzurePublicCloud \\\\\\n --set pullBinding.workloadIdentityClientId=\\\"${IMAGE_PULLER_MI_CLIENT_ID}\\\" \\\\\\n --set pullBinding.workloadIdentityTenantId=\\\"${IMAGE_PULLER_MI_TENANT_ID}\\\" \\\\\\n --set pullBinding.registry=arohcpsvcdev.azurecr.io \\\\\\n --set pullBinding.scope=repository:app-sre/aro-hcp-clusters-service:pull \\\\\\n --set managedIdentitiesDataPlaneAudienceResource=https://dummy.org \\\\\\n --set tracing.address=http://ingest.observability:4318 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxSurge=1 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxUnavailable=0 \\\\\\n --set provisionShardClusterLimit=100 \\\\\\n --set ocpVersions.defaultVersion.version=4.20.8 \\\\\\n --set ocpVersions.channelGroups.stable.minVersion=4.19.0 \\\\\\n --set ocpVersions.channelGroups.stable.maxVersion= \\\\\\n --set ocpVersions.channelGroups.candidate.minVersion=4.19.7 \\\\\\n --set ocpVersions.channelGroups.candidate.maxVersion= \\\\\\n --set ocpVersions.channelGroups.nightly.minVersion=4.19.0-0.nightly-20200101 \\\\\\n --set ocpVersions.channelGroups.nightly.maxVersion= \\\\\\n --set azureOperatorsMI.roleSetName=dev \\\\\\n --set deployment.zoneCount=4\\nnamespace/clusters-service created\\nnamespace/clusters-service labeled\\nRun Helm upgrade with release name cluster-service in namespace clusters-service\\nRelease \\\"cluster-service\\\" does not exist. Installing it now.\\nError: Unauthorized\\nmake[2]: *** [Makefile:32: deploy] Error 1\\nmake[2]: Leaving directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\n exit status 2\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40","failure_pattern":"make[2]: *** [Makefile:32: deploy] Error 1","search_query":"make[2]: *** [Makefile:32: deploy] Error 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033601057233309696","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4454,"pr_state":"closed","pr_sha":"38f866ad42f9613c0978385116cd5b0c6df796c1","final_merged_sha":"38f866ad42f9613c0978385116cd5b0c6df796c1","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T17:47:08Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"095ff39cd5cbd311feed94cbb6ae83657f141f39898f9510fd9a944642b6c648","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033601057233309696","occurred_at":"2026-03-16T17:47:08Z","failed_at":"provision","signature_id":"7a7d0f5d0e8ab95b7b5e95c3c532a9322e5dd52c30984711b49e7a1f1a32e9d3","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/management/output","test_suite":"step graph","failure_text":"time=2026-03-16T18:28:02.813Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=management step=output description=\"Step output\\n Kind: ARM\\n Template: ../dev-infrastructure/templates/output-mgmt.bicep\\n Parameters: ../dev-infrastructure/configurations/output-mgmt.tmpl.bicepparam\"\ntime=2026-03-16T18:28:03.295Z level=DEBUG+2 msg=\"Waiting for existing deployment to complete\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=management step=output deployment=4d5cdb6c661fd7a2d0d2ebc735e3a3b5a279f779ac9ba11949d31a9b27d78ed9\ntime=2026-03-16T18:31:04.547Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=management step=output err=\"failed to run ARM step: failed to poll previously-existing deployment: timed out waiting for pre-existing deployment to finish\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a3901d9621d6c16ce658f06646595e4a1cded970925c8d80d869b6f2b3b95e95","failure_pattern":"failed to run ARM step: failed to poll previously-existing deployment: timed out waiting for pre-existing deployment to finish","search_query":"failed to run ARM step: failed to poll previously-existing deployment: timed out waiting for pre-existing deployment to finish"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a3901d9621d6c16ce658f06646595e4a1cded970925c8d80d869b6f2b3b95e95"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033596623266779136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4473,"pr_state":"closed","pr_sha":"6b3c5b8735eaf7e0b9de792b429e03a13f4df15d","final_merged_sha":"8b7576c81d01104dc238546ec2678952bf7afd61","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T17:29:49Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"29b0007ef68827cf770ae13447938144af8350199298ea8c124cafc8113afa65","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033596623266779136","occurred_at":"2026-03-16T17:29:49Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4449/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033595245152702464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4449,"pr_state":"closed","pr_sha":"3b95084d22c390ee56e53ed8653eb39c2499db41","final_merged_sha":"205984e6ef0d1b76b1e624468408854ec2a65a8c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T17:23:01Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"8e7de4d77cc3b86111af84b0c5b65ce883e945971b02196a1bca30d48dd25743","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4449/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033595245152702464","occurred_at":"2026-03-16T17:23:01Z","failed_at":"provision","signature_id":"5a16efce6856cb7ec3c289414cb5c6df861b61434d949293d8a83736a8fb478f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-16T18:03:17.403Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Shell\\n Command: make deploy\\n\"\ntime=2026-03-16T18:03:50.165Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Shell Step, failed to execute shell command: make[2]: Entering directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\nkubectl create namespace clusters-service --dry-run=client -o json | kubectl apply -f - \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_CLIENT_ID=767aa9f5-c1a3-468f-a085-c445483009cf \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_TENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nkubectl label namespace clusters-service \\\"istio.io/rev=prod-stable\\\" --overwrite=true \u0026\u0026 \\\\\\nAZURE_CS_MI_CLIENT_ID=8160dccb-38e2-4b9a-a40a-66e5cdb22ae3 \u0026\u0026 \\\\\\nTENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nOIDC_BLOB_SERVICE_ENDPOINT=https://arohcpprowoidcj2702464.blob.core.windows.net/ \u0026\u0026 \\\\\\nOIDC_ISSUER_BASE_ENDPOINT=https://arohcpprowoidcj2702464.z1.web.core.windows.net/ \u0026\u0026 \\\\\\nDB_HOST=$(if [ \\\"false\\\" = \\\"true\\\" ]; then az postgres flexible-server show -g hcp-underlay-prow-j2702464 -n arohcp-prow-dbcs-j2702464 --query fullyQualifiedDomainName -o tsv; else echo \\\"ocm-cs-db\\\"; fi) \u0026\u0026 \\\\\\nOVERRIDES=$(if [ \\\"false\\\" = \\\"true\\\" ]; then echo \\\"azuredb.values.yaml\\\"; else echo \\\"containerdb.values.yaml\\\"; fi) \u0026\u0026 \\\\\\nZONE_RESOURCE_ID=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j2702464/providers/Microsoft.Network/dnszones/j2702464.hcp.osadev.cloud \u0026\u0026 \\\\\\nCX_SECRETS_KV_URL=\\\"https://ah-prow-cx-j2702464-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\nCX_MI_KV_URL=\\\"https://ah-prow-mi-j2702464-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\n./hack/helm.sh cluster-service helm-charts/cluster-service clusters-service \\\\\\n -f helm-charts/cluster-service/${OVERRIDES} \\\\\\n --set serviceAccountName=clusters-service \\\\\\n --set environment=arohcpprow \\\\\\n --set azureCsMiClientId=${AZURE_CS_MI_CLIENT_ID} \\\\\\n --set oidcIssuerBlobServiceUrl=${OIDC_BLOB_SERVICE_ENDPOINT} \\\\\\n --set oidcIssuerBaseUrl=${OIDC_ISSUER_BASE_ENDPOINT} \\\\\\n --set tlsCertificatesIssuer=${TLS_CERTIFICATES_ISSUER} \\\\\\n --set tlsCertificatesEnabled=true \\\\\\n --set denyAssignments=disabled \\\\\\n --set tenantId=${TENANT_ID} \\\\\\n --set region=westus3 \\\\\\n --set serviceKeyvaultName=aro-hcp-dev-svc-kv \\\\\\n --set replicas=3 \\\\\\n --set imageRegistry=arohcpsvcdev.azurecr.io \\\\\\n --set imageRepository=app-sre/aro-hcp-clusters-service \\\\\\n --set imageDigest=sha256:6ec7ecb11515ea6f8be560c4c043b518ef5c5ccc2a0aaafb2a45ee79c25a9d80 \\\\\\n --set global.imageDigest=sha256:6ec7ecb11515ea6f8be560c4c043b518ef5c5ccc2a0aaafb2a45ee79c25a9d80 \\\\\\n --set azureFirstPartyApplicationClientId=b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 \\\\\\n --set batchProcessesDryRun=true \\\\\\n --set batchProcesses=\\\"\\\" \\\\\\n --set fpaCertName=firstPartyCert2 \\\\\\n --set ocpAcrResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev \\\\\\n --set ocpAcrUrl=arohcpocpdev.azurecr.io \\\\\\n --set shard.consumerName=\\\"hcp-underlay-j2702464-mgmt-1\\\" \\\\\\n --set shard.zoneResourceId=\\\"${ZONE_RESOURCE_ID}\\\" \\\\\\n --set shard.cxSecretsKeyVaultUrl=\\\"${CX_SECRETS_KV_URL}\\\" \\\\\\n --set shard.cxMiKeyVaultUrl=\\\"${CX_MI_KV_URL}\\\" \\\\\\n --set shard.cxSecretsKeyVaultMiClientId=\\\"fa7c648c-f2f7-442d-b2a1-27c3deaa829a\\\" \\\\\\n --set shard.maestroRestUrl=\\\"http://maestro.maestro.svc.cluster.local:8000\\\" \\\\\\n --set shard.maestroGrpUrl=\\\"maestro-grpc.maestro.svc.cluster.local:8090\\\" \\\\\\n --set shard.Id=$(python3 -c \\\"import uuid; print(uuid.uuid5(uuid.UUID('916f9976-e1c0-4afd-b84c-5d5c94fbeaed'), 'prow-j2702464-mgmt-1'))\\\") \\\\\\n --set shard.aksManagementClusterResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j2702464-mgmt-1/providers/Microsoft.ContainerService/managedClusters/prow-j2702464-mgmt-1 \\\\\\n --set databaseHost=${DB_HOST} \\\\\\n --set azureMiMockServicePrincipalPrincipalId=d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 \\\\\\n --set azureMiMockServicePrincipalClientId=e8723db7-9b9e-46a4-9f7d-64d75c3534f0 \\\\\\n --set azureMiMockServicePrincipalCertName=msiMockCert2 \\\\\\n --set azureArmHelperIdentityCertName=armHelperCert2 \\\\\\n --set azureArmHelperIdentityClientId=3331e670-0804-48e8-a086-6241671ddc93 \\\\\\n --set azureArmHelperMockFpaPrincipalId=47f69502-0065-4d9a-b19b-d403e183d2f4 \\\\\\n --set azureCloudEnvironmentName=AzurePublicCloud \\\\\\n --set pullBinding.workloadIdentityClientId=\\\"${IMAGE_PULLER_MI_CLIENT_ID}\\\" \\\\\\n --set pullBinding.workloadIdentityTenantId=\\\"${IMAGE_PULLER_MI_TENANT_ID}\\\" \\\\\\n --set pullBinding.registry=arohcpsvcdev.azurecr.io \\\\\\n --set pullBinding.scope=repository:app-sre/aro-hcp-clusters-service:pull \\\\\\n --set managedIdentitiesDataPlaneAudienceResource=https://dummy.org \\\\\\n --set tracing.address=http://ingest.observability:4318 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxSurge=1 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxUnavailable=0 \\\\\\n --set provisionShardClusterLimit=100 \\\\\\n --set ocpVersions.defaultVersion.version=4.20.8 \\\\\\n --set ocpVersions.channelGroups.stable.minVersion=4.19.0 \\\\\\n --set ocpVersions.channelGroups.stable.maxVersion= \\\\\\n --set ocpVersions.channelGroups.candidate.minVersion=4.19.7 \\\\\\n --set ocpVersions.channelGroups.candidate.maxVersion= \\\\\\n --set ocpVersions.channelGroups.nightly.minVersion=4.19.0-0.nightly-20200101 \\\\\\n --set ocpVersions.channelGroups.nightly.maxVersion= \\\\\\n --set azureOperatorsMI.roleSetName=dev \\\\\\n --set deployment.zoneCount=4\\nnamespace/clusters-service created\\nnamespace/clusters-service labeled\\nRun Helm upgrade with release name cluster-service in namespace clusters-service\\nRelease \\\"cluster-service\\\" does not exist. Installing it now.\\nError: 3 errors occurred:\\n\\t* Unauthorized\\n\\t* Unauthorized\\n\\t* Unauthorized\\n\\n\\nmake[2]: *** [Makefile:32: deploy] Error 1\\nmake[2]: Leaving directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\n exit status 2\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40","failure_pattern":"make[2]: *** [Makefile:32: deploy] Error 1","search_query":"make[2]: *** [Makefile:32: deploy] Error 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033591131404505088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4433,"pr_state":"closed","pr_sha":"599dd2c53ef9a79607254436be4a2ed8e0671e8c","final_merged_sha":"599dd2c53ef9a79607254436be4a2ed8e0671e8c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T17:06:53Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"44647d978cb12ddccb482c1479fd8a47116aec4bda8273be76ea3c1ff1012000","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033591131404505088","occurred_at":"2026-03-16T17:06:53Z","failed_at":"provision","signature_id":"5a16efce6856cb7ec3c289414cb5c6df861b61434d949293d8a83736a8fb478f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-16T17:33:01.743Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Shell\\n Command: make deploy\\n\"\ntime=2026-03-16T17:33:49.040Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Shell Step, failed to execute shell command: make[2]: Entering directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\nkubectl create namespace clusters-service --dry-run=client -o json | kubectl apply -f - \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_CLIENT_ID=0028379a-fe01-4056-b086-40cc00b73848 \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_TENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nkubectl label namespace clusters-service \\\"istio.io/rev=prod-stable\\\" --overwrite=true \u0026\u0026 \\\\\\nAZURE_CS_MI_CLIENT_ID=7a657233-48ff-4136-83db-075edbbd38a0 \u0026\u0026 \\\\\\nTENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nOIDC_BLOB_SERVICE_ENDPOINT=https://arohcpprowoidcj4505088.blob.core.windows.net/ \u0026\u0026 \\\\\\nOIDC_ISSUER_BASE_ENDPOINT=https://arohcpprowoidcj4505088.z1.web.core.windows.net/ \u0026\u0026 \\\\\\nDB_HOST=$(if [ \\\"false\\\" = \\\"true\\\" ]; then az postgres flexible-server show -g hcp-underlay-prow-j4505088 -n arohcp-prow-dbcs-j4505088 --query fullyQualifiedDomainName -o tsv; else echo \\\"ocm-cs-db\\\"; fi) \u0026\u0026 \\\\\\nOVERRIDES=$(if [ \\\"false\\\" = \\\"true\\\" ]; then echo \\\"azuredb.values.yaml\\\"; else echo \\\"containerdb.values.yaml\\\"; fi) \u0026\u0026 \\\\\\nZONE_RESOURCE_ID=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4505088/providers/Microsoft.Network/dnszones/j4505088.hcp.osadev.cloud \u0026\u0026 \\\\\\nCX_SECRETS_KV_URL=\\\"https://ah-prow-cx-j4505088-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\nCX_MI_KV_URL=\\\"https://ah-prow-mi-j4505088-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\n./hack/helm.sh cluster-service helm-charts/cluster-service clusters-service \\\\\\n -f helm-charts/cluster-service/${OVERRIDES} \\\\\\n --set serviceAccountName=clusters-service \\\\\\n --set environment=arohcpprow \\\\\\n --set azureCsMiClientId=${AZURE_CS_MI_CLIENT_ID} \\\\\\n --set oidcIssuerBlobServiceUrl=${OIDC_BLOB_SERVICE_ENDPOINT} \\\\\\n --set oidcIssuerBaseUrl=${OIDC_ISSUER_BASE_ENDPOINT} \\\\\\n --set tlsCertificatesIssuer=${TLS_CERTIFICATES_ISSUER} \\\\\\n --set tlsCertificatesEnabled=true \\\\\\n --set denyAssignments=disabled \\\\\\n --set tenantId=${TENANT_ID} \\\\\\n --set region=westus3 \\\\\\n --set serviceKeyvaultName=aro-hcp-dev-svc-kv \\\\\\n --set replicas=3 \\\\\\n --set imageRegistry=arohcpsvcdev.azurecr.io \\\\\\n --set imageRepository=app-sre/aro-hcp-clusters-service \\\\\\n --set imageDigest=sha256:6ec7ecb11515ea6f8be560c4c043b518ef5c5ccc2a0aaafb2a45ee79c25a9d80 \\\\\\n --set global.imageDigest=sha256:6ec7ecb11515ea6f8be560c4c043b518ef5c5ccc2a0aaafb2a45ee79c25a9d80 \\\\\\n --set azureFirstPartyApplicationClientId=b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 \\\\\\n --set batchProcessesDryRun=true \\\\\\n --set batchProcesses=\\\"\\\" \\\\\\n --set fpaCertName=firstPartyCert2 \\\\\\n --set ocpAcrResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev \\\\\\n --set ocpAcrUrl=arohcpocpdev.azurecr.io \\\\\\n --set shard.consumerName=\\\"hcp-underlay-j4505088-mgmt-1\\\" \\\\\\n --set shard.zoneResourceId=\\\"${ZONE_RESOURCE_ID}\\\" \\\\\\n --set shard.cxSecretsKeyVaultUrl=\\\"${CX_SECRETS_KV_URL}\\\" \\\\\\n --set shard.cxMiKeyVaultUrl=\\\"${CX_MI_KV_URL}\\\" \\\\\\n --set shard.cxSecretsKeyVaultMiClientId=\\\"20783b0c-a764-4c66-a123-d16f025c284e\\\" \\\\\\n --set shard.maestroRestUrl=\\\"http://maestro.maestro.svc.cluster.local:8000\\\" \\\\\\n --set shard.maestroGrpUrl=\\\"maestro-grpc.maestro.svc.cluster.local:8090\\\" \\\\\\n --set shard.Id=$(python3 -c \\\"import uuid; print(uuid.uuid5(uuid.UUID('916f9976-e1c0-4afd-b84c-5d5c94fbeaed'), 'prow-j4505088-mgmt-1'))\\\") \\\\\\n --set shard.aksManagementClusterResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4505088-mgmt-1/providers/Microsoft.ContainerService/managedClusters/prow-j4505088-mgmt-1 \\\\\\n --set databaseHost=${DB_HOST} \\\\\\n --set azureMiMockServicePrincipalPrincipalId=d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 \\\\\\n --set azureMiMockServicePrincipalClientId=e8723db7-9b9e-46a4-9f7d-64d75c3534f0 \\\\\\n --set azureMiMockServicePrincipalCertName=msiMockCert2 \\\\\\n --set azureArmHelperIdentityCertName=armHelperCert2 \\\\\\n --set azureArmHelperIdentityClientId=3331e670-0804-48e8-a086-6241671ddc93 \\\\\\n --set azureArmHelperMockFpaPrincipalId=47f69502-0065-4d9a-b19b-d403e183d2f4 \\\\\\n --set azureCloudEnvironmentName=AzurePublicCloud \\\\\\n --set pullBinding.workloadIdentityClientId=\\\"${IMAGE_PULLER_MI_CLIENT_ID}\\\" \\\\\\n --set pullBinding.workloadIdentityTenantId=\\\"${IMAGE_PULLER_MI_TENANT_ID}\\\" \\\\\\n --set pullBinding.registry=arohcpsvcdev.azurecr.io \\\\\\n --set pullBinding.scope=repository:app-sre/aro-hcp-clusters-service:pull \\\\\\n --set managedIdentitiesDataPlaneAudienceResource=https://dummy.org \\\\\\n --set tracing.address=http://ingest.observability:4318 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxSurge=1 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxUnavailable=0 \\\\\\n --set provisionShardClusterLimit=100 \\\\\\n --set ocpVersions.defaultVersion.version=4.20.8 \\\\\\n --set ocpVersions.channelGroups.stable.minVersion=4.19.0 \\\\\\n --set ocpVersions.channelGroups.stable.maxVersion= \\\\\\n --set ocpVersions.channelGroups.candidate.minVersion=4.19.7 \\\\\\n --set ocpVersions.channelGroups.candidate.maxVersion= \\\\\\n --set ocpVersions.channelGroups.nightly.minVersion=4.19.0-0.nightly-20200101 \\\\\\n --set ocpVersions.channelGroups.nightly.maxVersion= \\\\\\n --set azureOperatorsMI.roleSetName=dev \\\\\\n --set deployment.zoneCount=4\\nnamespace/clusters-service created\\nnamespace/clusters-service labeled\\nRun Helm upgrade with release name cluster-service in namespace clusters-service\\nRelease \\\"cluster-service\\\" does not exist. Installing it now.\\nError: Unauthorized\\nmake[2]: *** [Makefile:32: deploy] Error 1\\nmake[2]: Leaving directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\n exit status 2\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40","failure_pattern":"make[2]: *** [Makefile:32: deploy] Error 1","search_query":"make[2]: *** [Makefile:32: deploy] Error 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033587508578619392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4219,"pr_state":"closed","pr_sha":"ae94c9371ff50f253c579d18b34e3e1f397bab9a","final_merged_sha":"8550ece78eb685f5f0357a6937bf6b6e9f457543","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T16:53:38Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"0ea7756268cb04de78afbfca05ed3ab4b80374b6ad4dde75ab1553b00160de73","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033587508578619392","occurred_at":"2026-03-16T16:53:38Z","failed_at":"provision","signature_id":"186999f428b36f71426b58a1596fcd1d6391598e85a630dbb52ced5c67a51707","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-crds","test_suite":"step graph","failure_text":"time=2026-03-16T17:30:06.934Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds description=\"Step deploy-mce-crds\\n Kind: Helm\\n\"\ntime=2026-03-16T17:30:07.452Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds values=map[]\ntime=2026-03-16T17:30:07.452Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds\ntime=2026-03-16T17:30:08.243Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-crds err=\"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace multicluster-engine: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b95b9c9bf7b4253eeec2aaaf028bc5abbf4cac62468310084bd0fb345750a5b","failure_pattern":"failed to apply namespace multicluster-engine: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace multicluster-engine: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7b95b9c9bf7b4253eeec2aaaf028bc5abbf4cac62468310084bd0fb345750a5b"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033582977929187328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4224,"pr_state":"closed","pr_sha":"89dbea9535e9da4e251791f532920158951c59f0","final_merged_sha":"0d2c310eee81cd65f33fd842930856c19dc160e3","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T16:35:38Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"9d93e4e6b411cb5542931ce83e5e85618e7ebbdace835a1c94bdc112b46e3a2c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033582977929187328","occurred_at":"2026-03-16T16:35:38Z","failed_at":"provision","signature_id":"7dc4fb9cfe9e0a6ce7afe61cb508e1ad4e8fa4df483ce758063f5f3f64cd897c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RouteMonitorOperator/management/default-ingresscontroller-hack","test_suite":"step graph","failure_text":"time=2026-03-16T17:00:54.253Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=default-ingresscontroller-hack description=\"Step default-ingresscontroller-hack\\n Kind: Shell\\n Command: make default-ingresscontroller-hack\\n\"\ntime=2026-03-16T17:00:55.925Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=default-ingresscontroller-hack err=\"failed to prepare kubeconfig: failed to ensure cluster admin role: failed to get current user object ID: failed to get current user: /me request is only valid with delegated authentication flow.\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1466406c48902bc9d663394fbe43cd72c554c24ca2f6aa7ec287ef1819329804","failure_pattern":"failed to prepare kubeconfig: failed to ensure cluster admin role: failed to get current user object ID: failed to get current user: /me request is only valid with delegated authentication flow.","search_query":"failed to prepare kubeconfig: failed to ensure cluster admin role: failed to get current user object ID: failed to get current user: /me request is only valid with delegated authentication flow."}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1466406c48902bc9d663394fbe43cd72c554c24ca2f6aa7ec287ef1819329804"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033581634162266112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4226,"pr_state":"closed","pr_sha":"3f588748fd29207330fe25d741a19c20e8d6fbb8","final_merged_sha":"138964dfc1d79a030eb9c07890259cb904e592a1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T16:30:17Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"24d5ff77262c64f50196d01cf257df7a6a7936ee4b7d7b8b1fe6a21f5d36409a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033581634162266112","occurred_at":"2026-03-16T16:30:17Z","failed_at":"provision","signature_id":"031c54fa35aec9e0c04cce1b936b80aba069b2956505852e58193ab6a029a41c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RouteMonitorOperator/management/deploy","test_suite":"step graph","failure_text":"time=2026-03-16T16:55:02.720Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-16T16:55:03.362Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy values=\"map[imageDigestBlackbox:sha256:b04a9fef4fa086a02fc7fcd8dcdbc4b7b35cc30cdee860fdc6a19dd8b208d63e imageDigestManager:sha256:5e94ff37bcb8c1ebebce0c7c1f33cef8e7cf163f1b0baf5b50c9b771ca112846 imageDigestOperator:sha256:5e94ff37bcb8c1ebebce0c7c1f33cef8e7cf163f1b0baf5b50c9b771ca112846 imageRegistryBlackbox:arohcpsvcdev.azurecr.io imageRegistryManager:arohcpsvcdev.azurecr.io imageRegistryOperator:arohcpsvcdev.azurecr.io imageRepositoryBlackbox:prometheus/blackbox-exporter imageRepositoryManager:app-sre/route-monitor-operator imageRepositoryOperator:app-sre/route-monitor-operator]\"\ntime=2026-03-16T16:55:03.362Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy\ntime=2026-03-16T16:55:03.643Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy namespace=openshift-route-monitor-operator\ntime=2026-03-16T16:55:03.944Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy dryRun=false\ntime=2026-03-16T16:55:03.944Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy\ntime=2026-03-16T16:55:04.673Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy numVersions=0\ntime=2026-03-16T16:55:04.782Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy err=\"\\\"route-monitor-operator\\\" has no deployed releases\"\ntime=2026-03-16T16:55:04.782Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy\ntime=2026-03-16T16:55:04.782Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy\ntime=2026-03-16T16:55:04.891Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy err=\"failed to get status for release route-monitor-operator: release: not found\"\ntime=2026-03-16T16:55:04.891Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy\ntime=2026-03-16T16:55:04.891Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=management step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: \\\"route-monitor-operator\\\" has no deployed releases\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"29c7f7fdf70af226b1ee8e85aa1778b2b77e423a9c303a15723bcf43adf003a9","failure_pattern":"\"route-monitor-operator\" has no deployed releases","search_query":"error running Helm release deployment Step, failed to deploy helm release: \\\"route-monitor-operator\\\" has no deployed releases"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["29c7f7fdf70af226b1ee8e85aa1778b2b77e423a9c303a15723bcf43adf003a9"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033576608308137984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4433,"pr_state":"closed","pr_sha":"599dd2c53ef9a79607254436be4a2ed8e0671e8c","final_merged_sha":"599dd2c53ef9a79607254436be4a2ed8e0671e8c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T16:10:20Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"f0c95f2123f9ceace04a7d132d1751135f57a09047d906d8d60dd26d2c4ab422","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033576608308137984","occurred_at":"2026-03-16T16:10:20Z","failed_at":"provision","signature_id":"a5b5f71a061e60f11c9c511d431eae042d41346983d1442cca50d855704dabbe","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/prometheus","test_suite":"step graph","failure_text":"time=2026-03-16T16:26:24.901Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-03-16T16:26:25.637Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[customResourceState:map[config:map[spec:map[resources:[map[groupVersionKind:map[group:hypershift.openshift.io kind:HostedCluster version:v1beta1] labelsFromPath:map[_id:[spec clusterID] api_url:[spec services [service=APIServer] servicePublishingStrategy route hostname] name:[metadata name] namespace:[metadata namespace]] metricNamePrefix:hostedClusterAPI metrics:[map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=KubeAPIServerAvailable] status]] type:StateSet] help:KubeAPIServer availability condition name:kubeapiserver_available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Available] status]] type:StateSet] help:Cluster availability condition name:available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Progressing] status]] type:StateSet] help:Cluster progressing condition name:progressing]]]]]] create:true enabled:true key:config.yaml name:kube-state-metric-custom-resource-config] image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c] rbac:map[extraRules:[map[apiGroups:[hypershift.openshift.io] resources:[hostedclusters] verbs:[list watch]]]]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:d7e8257f8d8bce64b6df55f81fba92011a6a77269b3350f8b997b152af348dba]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:51a9932a-8ace-44d0-9e3f-f999b80392d3]] prometheusSpec:map[externalLabels:map[cluster:prow-j8137984-mgmt-1] hcpRemoteWriteUrl:https://msprom-westus3-prow-j8137984-mgmt-1-63b2.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-9852b4015418490d932705247af74e0f/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:dca4b35f73046dc4803c0160c2e6be5a1736a716b26ef35590864d36dfdf950a] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j8137984-mgmt-1-63b2.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-50d428240f7d4caaac3e60d6f99d85b5/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-16T16:26:25.638Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T16:26:26.084Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus namespace=prometheus\ntime=2026-03-16T16:26:26.477Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus dryRun=false\ntime=2026-03-16T16:26:26.477Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T16:26:26.616Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T16:26:47.452Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"failed pre-install: warning: Hook pre-install hcp-prometheus/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml failed: Unauthorized\"\ntime=2026-03-16T16:26:47.452Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T16:26:47.452Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T16:27:00.535Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus release=arohcp-monitor namespace=prometheus status=failed description=\"Release \\\"arohcp-monitor\\\" failed: failed pre-install: warning: Hook pre-install hcp-prometheus/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml failed: Unauthorized\" values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[customResourceState:map[config:map[spec:map[resources:[map[groupVersionKind:map[group:hypershift.openshift.io kind:HostedCluster version:v1beta1] labelsFromPath:map[_id:[spec clusterID] api_url:[spec services [service=APIServer] servicePublishingStrategy route hostname] name:[metadata name] namespace:[metadata namespace]] metricNamePrefix:hostedClusterAPI metrics:[map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=KubeAPIServerAvailable] status]] type:StateSet] help:KubeAPIServer availability condition name:kubeapiserver_available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Available] status]] type:StateSet] help:Cluster availability condition name:available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Progressing] status]] type:StateSet] help:Cluster progressing condition name:progressing]]]]]] create:true enabled:true key:config.yaml name:kube-state-metric-custom-resource-config] image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c] rbac:map[extraRules:[map[apiGroups:[hypershift.openshift.io] resources:[hostedclusters] verbs:[list watch]]]]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:d7e8257f8d8bce64b6df55f81fba92011a6a77269b3350f8b997b152af348dba]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:51a9932a-8ace-44d0-9e3f-f999b80392d3]] prometheusSpec:map[externalLabels:map[cluster:prow-j8137984-mgmt-1] hcpRemoteWriteUrl:https://msprom-westus3-prow-j8137984-mgmt-1-63b2.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-9852b4015418490d932705247af74e0f/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:dca4b35f73046dc4803c0160c2e6be5a1736a716b26ef35590864d36dfdf950a] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j8137984-mgmt-1-63b2.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-50d428240f7d4caaac3e60d6f99d85b5/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-16T16:27:00.536Z level=INFO msg=\"No resource info available in release, skipping resource diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus infoNil=false resourceCount=0\ntime=2026-03-16T16:27:00.536Z level=INFO msg=\"Kusto deep link for namespace-level troubleshooting\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus kustoLinkNamespace=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/3yOvaqDQBSEe59i2EpBRb0XL1zwDVKmsgnrOqgh+8PuBgn48MEUJlXgFIePGeZT1kS5GPqTnUKyYZ3pibhohii1w8C4kgbpKCN3nIqmatqi+inq9ly3/81+vchQlviS+f3rRZYdA0ZqBicVL/uHroNw3mrGmfcgkg3O2ytVfJvkuNkph7Pjq5JssH6kx/A4GGRQ+Ye7DOoZAAD///9xGUHfAAAA\"\ntime=2026-03-16T16:27:00.537Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-16T16:27:00.537Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: failed pre-install: warning: Hook pre-install hcp-prometheus/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml failed: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5cfdaad31b0722b6f33b19cd419a6bf5cf43c46a79b5dfab5dbdce5daf0f63d6","failure_pattern":"failed pre-install: warning: Hook pre-install hcp-prometheus/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml failed: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed pre-install: warning: Hook pre-install hcp-prometheus/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml failed: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["5cfdaad31b0722b6f33b19cd419a6bf5cf43c46a79b5dfab5dbdce5daf0f63d6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033570194168745984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4473,"pr_state":"closed","pr_sha":"3cda2b602484ff73b166151edce149ca0758f985","final_merged_sha":"8b7576c81d01104dc238546ec2678952bf7afd61","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T15:44:50Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"5a9f3e76af046d4079593c6a286c99e7736e973daae998e2554c9bd4843546f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033570194168745984","occurred_at":"2026-03-16T15:44:50Z","failed_at":"provision","signature_id":"ed643408e7d9e12b4f1d47ca465cdb880030600970d562392c3859481c2bc608","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.AdminAPI/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-16T16:26:27.268Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-16T16:26:27.834Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=service step=deploy values=\"map[audit:map[connectSocket:false] azure:map[clientId:9b335788-9698-4c2e-b8b6-e774d43dd7a8 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] clustersService:map[namespace:clusters-service] cosmosDB:map[name:arohcpprow-rp-j8745984 url:https://arohcpprow-rp-j8745984.documents.azure.com:443/] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:admin-api-cert-prow-j8745984] deployment:map[limits:map[cpu:1 memory:1Gi] replicas:2 requests:map[cpu:200m memory:512Mi] zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 keyVaultName:aro-hcp-dev-svc-kv] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:admin-api-https] host:admin.j8745984.hcpsvc.osadev.cloud] image:map[digest:sha256:765bfaa0fea5572f7c547c3041dda88339289f0a850136bf92b4d880bb2d886f registry:arohcpsvcdev.azurecr.io repository:ci-op-yg9z9inx/pipeline] kusto:map[endpoint:https://hcp-dev-us-2.eastus2.kusto.windows.net] location:westus3 mise:map[deploy:false policyLabel:Geneva Actions] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-yg9z9inx/pipeline:pull workloadIdentityClientId:4e03dc43-0ff5-4d72-979f-67d7f7f56198 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:admin-api port:8443] serviceAccount:map[name:admin-api workloadIdentityClientId:163d854a-7dae-475f-9cc3-141cdae7b1c5 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] sessiongate:map[maxSessionTTL:24h minSessionTTL:1m mise:map[policyLabel:Session Gate] namespace:sessiongate servicePort:8080] virtualService:map[host:admin.j8745984.hcpsvc.osadev.cloud]]\"\ntime=2026-03-16T16:26:27.835Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=service step=deploy\ntime=2026-03-16T16:26:28.066Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=service step=deploy namespace=aro-hcp-admin-api\ntime=2026-03-16T16:26:28.357Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=service step=deploy dryRun=false\ntime=2026-03-16T16:26:28.357Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=service step=deploy\ntime=2026-03-16T16:26:28.537Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=service step=deploy\ntime=2026-03-16T16:26:35.567Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=service step=deploy err=\"Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.37:15017, code 500: 500 Internal Server Error\"\ntime=2026-03-16T16:26:35.567Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=service step=deploy\ntime=2026-03-16T16:26:35.567Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=service step=deploy\ntime=2026-03-16T16:26:36.947Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=service step=deploy release=admin-api namespace=aro-hcp-admin-api status=failed description=\"Release \\\"admin-api\\\" failed: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.37:15017, code 500: 500 Internal Server Error\" values=\"map[audit:map[connectSocket:false] azure:map[clientId:9b335788-9698-4c2e-b8b6-e774d43dd7a8 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] clustersService:map[namespace:clusters-service] cosmosDB:map[name:arohcpprow-rp-j8745984 url:https://arohcpprow-rp-j8745984.documents.azure.com:443/] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:admin-api-cert-prow-j8745984] deployment:map[limits:map[cpu:1 memory:1Gi] replicas:2 requests:map[cpu:200m memory:512Mi] zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 keyVaultName:aro-hcp-dev-svc-kv] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:admin-api-https] host:admin.j8745984.hcpsvc.osadev.cloud] image:map[digest:sha256:765bfaa0fea5572f7c547c3041dda88339289f0a850136bf92b4d880bb2d886f registry:arohcpsvcdev.azurecr.io repository:ci-op-yg9z9inx/pipeline] kusto:map[endpoint:https://hcp-dev-us-2.eastus2.kusto.windows.net] location:westus3 mise:map[deploy:false policyLabel:Geneva Actions] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-yg9z9inx/pipeline:pull workloadIdentityClientId:4e03dc43-0ff5-4d72-979f-67d7f7f56198 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:admin-api port:8443] serviceAccount:map[name:admin-api workloadIdentityClientId:163d854a-7dae-475f-9cc3-141cdae7b1c5 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] sessiongate:map[maxSessionTTL:24h minSessionTTL:1m mise:map[policyLabel:Session Gate] namespace:sessiongate servicePort:8080] virtualService:map[host:admin.j8745984.hcpsvc.osadev.cloud]]\"\ntime=2026-03-16T16:26:36.949Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5yTQWvbQBCF7/oVg04xaEPiUFNcckjTQqG0mNT0kNLDancsT7zaXWZGDgH/+CIbJ2qJjdPrm/e9mZF2ioAKjJI6dihwDd6qVVsHPEv1Azr9StFPRZliU8FO+m5bfJZwjXGrSLZuL49+FQAA5W3oRJHvUsCPFD3FpqygtL6laGwmIyhCKZq2abWvlNWOu3E860IYMLkLwdSDDE5m6bJ5ztqjX+bz2V3qFP/qdBT5gY5RZ5zW5JFvgxXp/YtsjUPW/2EHK7q8FVZiSJSSodgw9q59AK/J4bcUSRMf6HV0gC1/41zqor5l6S33qse0qExOTg44qeMnzCE9tfiWKQ8w2/9CC3JW0TAuGGWJfPQ7/yTWzobXpjbrA5sWvz8Uq65Gjqgon/uXLsUGHpfICEotito2Q436iBjhzFvFXj4rxxfjibm4MpeT+eVkOp5Mx+/vyxGcn8MRz9W7+3I0KjbwkCjCiqK/phiRB/eZIryc5fAe/z3EYgOZU199GbSCnPzOfFpIBSE1FbQoYps+MrFHhvppsLxHcX8CAAD//wRnC6JFBAAA\"\ntime=2026-03-16T16:26:36.952Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=service step=deploy\ntime=2026-03-16T16:26:36.953Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=service step=deploy step=deploy error=\"error running Helm release deployment Step, failed to deploy helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.37:15017, code 500: 500 Internal Server Error\"\ntime=2026-03-16T16:27:36.957Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-16T16:27:37.532Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=service step=deploy values=\"map[audit:map[connectSocket:false] azure:map[clientId:9b335788-9698-4c2e-b8b6-e774d43dd7a8 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] clustersService:map[namespace:clusters-service] cosmosDB:map[name:arohcpprow-rp-j8745984 url:https://arohcpprow-rp-j8745984.documents.azure.com:443/] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:admin-api-cert-prow-j8745984] deployment:map[limits:map[cpu:1 memory:1Gi] replicas:2 requests:map[cpu:200m memory:512Mi] zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 keyVaultName:aro-hcp-dev-svc-kv] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:admin-api-https] host:admin.j8745984.hcpsvc.osadev.cloud] image:map[digest:sha256:765bfaa0fea5572f7c547c3041dda88339289f0a850136bf92b4d880bb2d886f registry:arohcpsvcdev.azurecr.io repository:ci-op-yg9z9inx/pipeline] kusto:map[endpoint:https://hcp-dev-us-2.eastus2.kusto.windows.net] location:westus3 mise:map[deploy:false policyLabel:Geneva Actions] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-yg9z9inx/pipeline:pull workloadIdentityClientId:4e03dc43-0ff5-4d72-979f-67d7f7f56198 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:admin-api port:8443] serviceAccount:map[name:admin-api workloadIdentityClientId:163d854a-7dae-475f-9cc3-141cdae7b1c5 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] sessiongate:map[maxSessionTTL:24h minSessionTTL:1m mise:map[policyLabel:Session Gate] namespace:sessiongate servicePort:8080] virtualService:map[host:admin.j8745984.hcpsvc.osadev.cloud]]\"\ntime=2026-03-16T16:27:37.532Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=service step=deploy\ntime=2026-03-16T16:27:37.627Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace aro-hcp-admin-api: an error on the server (\\\"Internal Server Error: \\\\\\\"/api/v1/namespaces/aro-hcp-admin-api?fieldManager=templatize\u0026amp;force=false\\\\\\\": Post \\\\\\\"https://guard.69b82b1a32e3a9000188c805.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": proxy error from localhost:9443 while dialing guard.69b82b1a32e3a9000188c805.svc.cluster.local:443, code 503: 503 Service Unavailable\\\") has prevented the request from succeeding (patch namespaces aro-hcp-admin-api)\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"528e7785c0af294e0105f52c61cbbc134c2a11f1f82ff176d84be2ede7ee0bee","failure_pattern":"failed to apply namespace aro-hcp-admin-api: an error on the server (\"Internal Server Error: \\\\\"/api/v1/namespaces/aro-hcp-admin-api?fieldManager=templatize\u0026amp;force=false\\\\\": Post \\\\\"\u003curl\u003e\": proxy error from localhost:9443 while dialing","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace aro-hcp-admin-api: an error on the server (\\\"Internal Server Error: \\\\\\\"/api/v1/namespaces/aro-hcp-admin-api?fieldManager=templatize\u0026amp;force=false\\\\\\\": Post \\\\\\\"https://guard.69b82b1a32e3a9000188c805.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": proxy error from localhost:9443 while dialing guard.69b82b1a32e3a9000188c805.svc.cluster.local:443, code 503: 503 Service Unavailable\\\") has prevented the request from succeeding (patch namespaces aro-hcp-admin-api)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["528e7785c0af294e0105f52c61cbbc134c2a11f1f82ff176d84be2ede7ee0bee"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033565546229796864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4393,"pr_state":"open","pr_sha":"dcec1a0f0e9fc5822919f091837faed741634449","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T15:26:22Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"31e2d6fd153451f2cea6bb9e9730400f45361345c80acafe155e760ecc59215c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033565546229796864","occurred_at":"2026-03-16T15:26:22Z","failed_at":"provision","signature_id":"692850ccec3627f6f18c0069e48528a9b9f4e27dd84dcc0a7a087aa7c2d7afa7","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce","test_suite":"step graph","failure_text":"time=2026-03-16T16:22:40.958Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce description=\"Step deploy-mce\\n Kind: Helm\\n\"\ntime=2026-03-16T16:22:41.564Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f imageRegistry:arohcpsvcdev.azurecr.io imageRootRepository:acm-d-cache]\"\ntime=2026-03-16T16:22:41.564Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-16T16:22:41.623Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce namespace=multicluster-engine\ntime=2026-03-16T16:22:41.738Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce dryRun=false\ntime=2026-03-16T16:22:41.738Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-16T16:22:41.855Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-16T16:24:41.768Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce err=\"failed post-install: Unauthorized\"\ntime=2026-03-16T16:24:41.768Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-16T16:24:41.768Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-16T16:24:45.935Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce release=mce namespace=multicluster-engine status=failed description=\"Release \\\"mce\\\" failed: failed post-install: Unauthorized\" values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f imageRegistry:arohcpsvcdev.azurecr.io imageRootRepository:acm-d-cache]\"\ntime=2026-03-16T16:24:45.936Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/7SUS2/TQBSF9/4VV161kqdqXCcpQV3wWiFYACsQi/H4xJ56Xp1HIlf58chNQ6JKIRGC7Zmjz+Nz555MIZJHsMkLBLqjhkceea1wYet7iPhRmmYRopemLWgrfeYavyWsYJ6U4LjYyZc/MiKi/Cv8Sgq8EcImE/OCcp1UlEKlEOEZTCsNmHXwPFp/5DwvdjDhcRLC1qg7a/uzYdbBsJ1Jc8NbaJjIpOYtmEtKMeHRwETJVTiBfbdVv1iFt9I00rTH7utSp6ub+Wrq7KtqsJUzrmnn02nFZTn36lFX+jZNH7p53YyI3QdG8jFkWQ0uzfvlEMPgy0llpLSdR0r9bYle3yzXj76zKuDhGGKfztPczs2ahT/6X1A/WSOfp/3sU3IJMQgFFiKPYBrRSxHYqjw/7v+Q84kJ/ru438MpO4yv7m9XJPv5OutTDW8QET6MOxmyDa07eFCUGiFy7ahGXAOGLhoeMcoXeXldztj1DZvMvk1mi7JcVJPv+SVdXdFRT7X1XGYburfSUC9NcyeNgT9oEmtoXyCHzfGyMrINOW/H0/1FC3K22ZrPgxSkbFuQRgi8HZHWN/BUDwc/3yCIXwEAAP//5ifO5u8EAAA=\"\ntime=2026-03-16T16:24:45.937Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-16T16:24:45.938Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce err=\"error running Helm release deployment Step, failed to deploy helm release: failed post-install: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82e6580bc32e56aa7378422fd2960d02a2eebbb9fbfbc08994384c40137bbab6","failure_pattern":"failed post-install: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed post-install: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["82e6580bc32e56aa7378422fd2960d02a2eebbb9fbfbc08994384c40137bbab6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033563968341020672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4457,"pr_state":"open","pr_sha":"c6d34c6bc54bf82f40b7a2b9e91f43b977c11e31","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T15:20:06Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"5bfd2583b928561fa0aa3ead846ba3d146b2b79dbc5debce42d6dda910d28b95","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033563968341020672","occurred_at":"2026-03-16T15:20:06Z","failed_at":"provision","signature_id":"406d7bf8017775214d3988b7aa96323527511df891704af205ce55331783d9c4","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy-cs-debug-jobs","test_suite":"step graph","failure_text":"time=2026-03-16T16:26:33.542Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy-cs-debug-jobs description=\"Step deploy-cs-debug-jobs\\n Kind: Shell\\n Command: make deploy-cs-debug-jobs\\n\"\ntime=2026-03-16T16:26:36.717Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy-cs-debug-jobs err=\"error running Shell Step, failed to execute shell command: make[2]: Entering directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\n./hack/helm.sh cs-debug-jobs helm-charts/cs-debug-jobs clusters-service \\\\\\n --set imageDigest=sha256:6ec7ecb11515ea6f8be560c4c043b518ef5c5ccc2a0aaafb2a45ee79c25a9d80 \\\\\\n --set deployDebugJobs=false\\nRun Helm upgrade with release name cs-debug-jobs in namespace clusters-service\\nRelease \\\"cs-debug-jobs\\\" does not exist. Installing it now.\\nE0316 16:26:36.708578 2397 memcache.go:265] \\\"Unhandled Error\\\" err=\\\"couldn't get current server API group list: the server has asked for the client to provide credentials\\\" logger=\\\"UnhandledError\\\"\\nError: could not get apiVersions from Kubernetes: could not get apiVersions from Kubernetes: the server has asked for the client to provide credentials\\nmake[2]: *** [Makefile:106: deploy-cs-debug-jobs] Error 1\\nmake[2]: Leaving directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\n exit status 2\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"56027315c26b664b21966f509b496e1c0ae9a3979ef382f30b8ff6507f2fea13","failure_pattern":"make[2]: *** [Makefile:106: deploy-cs-debug-jobs] Error 1","search_query":"make[2]: *** [Makefile:106: deploy-cs-debug-jobs] Error 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["56027315c26b664b21966f509b496e1c0ae9a3979ef382f30b8ff6507f2fea13"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4455/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033563732600164352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4455,"pr_state":"closed","pr_sha":"7b9532af16d82b9a9786a1179527a9bf0cc955bb","final_merged_sha":"7b9532af16d82b9a9786a1179527a9bf0cc955bb","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T15:19:10Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"1b1bcef0290f10b0ff47c1c52117e2646f8fe96d8f4c2b6969bc20872d64eaa9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4455/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033563732600164352","occurred_at":"2026-03-16T15:19:10Z","failed_at":"provision","signature_id":"692850ccec3627f6f18c0069e48528a9b9f4e27dd84dcc0a7a087aa7c2d7afa7","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce","test_suite":"step graph","failure_text":"time=2026-03-16T16:21:34.726Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce description=\"Step deploy-mce\\n Kind: Helm\\n\"\ntime=2026-03-16T16:21:35.583Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f imageRegistry:arohcpsvcdev.azurecr.io imageRootRepository:acm-d-cache]\"\ntime=2026-03-16T16:21:35.583Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-16T16:21:35.646Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce namespace=multicluster-engine\ntime=2026-03-16T16:21:35.754Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce dryRun=false\ntime=2026-03-16T16:21:35.754Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-16T16:21:35.861Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-16T16:23:37.789Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce err=\"failed post-install: Unauthorized\"\ntime=2026-03-16T16:23:37.789Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-16T16:23:37.789Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-16T16:23:39.303Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce err=\"failed to get status for release mce: kubernetes cluster unreachable: the server has asked for the client to provide credentials\"\ntime=2026-03-16T16:23:39.303Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-16T16:23:39.303Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce err=\"error running Helm release deployment Step, failed to deploy helm release: failed post-install: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82e6580bc32e56aa7378422fd2960d02a2eebbb9fbfbc08994384c40137bbab6","failure_pattern":"failed post-install: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed post-install: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["82e6580bc32e56aa7378422fd2960d02a2eebbb9fbfbc08994384c40137bbab6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4449/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033558327891333120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4449,"pr_state":"closed","pr_sha":"3b95084d22c390ee56e53ed8653eb39c2499db41","final_merged_sha":"205984e6ef0d1b76b1e624468408854ec2a65a8c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T14:57:41Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"5137ea7f9afa98b84ee81b33cfe288924f9d5cbafeba62e2a5896727d9385061","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4449/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033558327891333120","occurred_at":"2026-03-16T14:57:41Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4481/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033557983643832320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4481,"pr_state":"closed","pr_sha":"3d3fe8e4a80bc62b359d5e0e012605263fbe3fc6","final_merged_sha":"a0ece52bde4e4930997796ae17d728b08f758b6b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T14:56:14Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"97e10179eb897bd6f8eada430c73bbcac7761a82af15dcc824108a3c679944cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4481/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033557983643832320","occurred_at":"2026-03-16T14:56:14Z","failed_at":"provision","signature_id":"01371870d1c1bbd605f5f541d9708d5433ef59976a00c4ec4810955526889a38","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-config","test_suite":"step graph","failure_text":"time=2026-03-16T16:24:00.786Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config description=\"Step deploy-mce-config\\n Kind: Helm\\n\"\ntime=2026-03-16T16:24:01.400Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f global:map[registryOverride:arohcpsvcdev.azurecr.io/acm-d-cache] localCluster:map[addonConfig:map[applicationManagerEnabled:false certPolicyControllerEnabled:false policyControllerEnabled:true searchCollectorEnabled:false] kubeApiUrl:https://kubernetes.default.svc] policy:map[cluster-lifecycle:map[hubconfig:map[replicaCount:2]] grc:map[hubconfig:map[replicaCount:2]]]]\"\ntime=2026-03-16T16:24:01.400Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-16T16:24:01.504Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace multicluster-engine: an error on the server (\\\"Internal Server Error: \\\\\\\"/api/v1/namespaces/multicluster-engine?fieldManager=templatize\u0026amp;force=false\\\\\\\": Post \\\\\\\"https://guard.69b82a8e54541c000131182e.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": proxy error from localhost:9443 while dialing guard.69b82a8e54541c000131182e.svc.cluster.local:443, code 503: 503 Service Unavailable\\\") has prevented the request from succeeding (patch namespaces multicluster-engine)\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c0cd85cfb126ea2bfa65f4cb0d56b1c8fb3d07e37b777bea20639a906e9ba90b","failure_pattern":"failed to apply namespace multicluster-engine: an error on the server (\"Internal Server Error: \\\\\"/api/v1/namespaces/multicluster-engine?fieldManager=templatize\u0026amp;force=false\\\\\": Post \\\\\"\u003curl\u003e\": proxy error from localhost:9443 while dialing","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace multicluster-engine: an error on the server (\\\"Internal Server Error: \\\\\\\"/api/v1/namespaces/multicluster-engine?fieldManager=templatize\u0026amp;force=false\\\\\\\": Post \\\\\\\"https://guard.69b82a8e54541c000131182e.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": proxy error from localhost:9443 while dialing guard.69b82a8e54541c000131182e.svc.cluster.local:443, code 503: 503 Service Unavailable\\\") has prevented the request from succeeding (patch namespaces multicluster-engine)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c0cd85cfb126ea2bfa65f4cb0d56b1c8fb3d07e37b777bea20639a906e9ba90b"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4241/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033556485073539072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4241,"pr_state":"closed","pr_sha":"f0eb454d49936e73b94a7710cef63a611155b521","final_merged_sha":"f0eb454d49936e73b94a7710cef63a611155b521","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T14:49:49Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"506d89a8a325af86072a12847d007a42867d3ccb99763646c0d9ad50e8f2fad1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4241/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033556485073539072","occurred_at":"2026-03-16T14:49:49Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033555709936799744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4226,"pr_state":"closed","pr_sha":"3f588748fd29207330fe25d741a19c20e8d6fbb8","final_merged_sha":"138964dfc1d79a030eb9c07890259cb904e592a1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T14:47:17Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"c2e006bff44b95cd898e34eefb643a280e7e7270deb199e0c0f1b23f8b103167","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033555709936799744","occurred_at":"2026-03-16T14:47:17Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033550886466228224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4224,"pr_state":"closed","pr_sha":"89dbea9535e9da4e251791f532920158951c59f0","final_merged_sha":"0d2c310eee81cd65f33fd842930856c19dc160e3","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T14:28:07Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"93d81bd7154a41b7985a9acff3510bf0fd40e231852e0f2143b6457a26da2cd8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033550886466228224","occurred_at":"2026-03-16T14:28:07Z","failed_at":"provision","signature_id":"c027c4d12f6a3d02006d0282f3d5b1020328ed77f7bf43d76c7a5e7566e656ac","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-03-16T15:55:01.252Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-03-16T15:55:03.096Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-03-16T15:55:04.692Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=7a202efbc0acb584cd9967ea2396ce403b176cb14a71bf6db367a49493f4a335\ntime=2026-03-16T16:09:08.071Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j6228224-svc/providers/Microsoft.Resources/deployments/7a202efbc0acb584cd9967ea2396ce403b176cb14a71bf6db367a49493f4a335/operationStatuses/08584279303816608430\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/frontend-cert-zqyt3pxcol57e\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deploymentScripts/newCert-frontend-cert-prow-j6228224\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentScriptOperationFailed\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The container group 'dbon63o3dfre6azscripts' is still transitioning, please retry later.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e6de0b2463ca9e1599d761550cbd2bcc4aac3f0e89a91386c261124d25fc09b2","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentScriptOperationFailed; detail message The container group '\u003cid\u003e' is still transitioning, please retry later.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e6de0b2463ca9e1599d761550cbd2bcc4aac3f0e89a91386c261124d25fc09b2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4461/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033546562403045376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4461,"pr_state":"closed","pr_sha":"421438d23c10f9e39e468effb1aad24a3674531d","final_merged_sha":"9baa329552156f993c1fc5b382b78423cd754fef","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T14:10:55Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"0f5aecab2c3dfb440ffc98d426038e71f3168a092dfbe198eece73e46f044a82","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4461/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033546562403045376","occurred_at":"2026-03-16T14:10:55Z","failed_at":"e2e","signature_id":"77a8f6c6add5cef6e1133c424143132891764a2c8e50cf325fd57ae7d3bf07ec","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.007s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000d776a0\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.007s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000d776a0\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}},{"row_id":"243d29a799dcc96a17ca1993696ee5defe7aa7999e16884b2e4e4ff550eb6b8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4461/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033546562403045376","occurred_at":"2026-03-16T14:10:55Z","failed_at":"e2e","signature_id":"300de6423494fee6be9a3fd2729c1214cab2d0a02daeed941d811be0285618c3","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243]: Unexpected error:\n \u003c*errors.joinError | 0xc0012701e0\u003e: \n failed to cleanup resource group: failed listing hcp clusters in resourcegroup=\"basic-cluster-j2w6hk\": GET https://rp.j3045376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-j2w6hk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n errs: [\n \u003c*fmt.wrapError | 0xc0009f48c0\u003e{\n msg: \"failed to cleanup resource group: failed listing hcp clusters in resourcegroup=\\\"basic-cluster-j2w6hk\\\": GET https://rp.j3045376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-j2w6hk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0009f4860\u003e{\n msg: \"failed listing hcp clusters in resourcegroup=\\\"basic-cluster-j2w6hk\\\": GET https://rp.j3045376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-j2w6hk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001102540\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"107\"],\n \"X-Envoy-Upstream-Service-Time\": [\"2669\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"ff126f3d-baf1-4954-a9ec-cc35a45b7494\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 16:32:56 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0009f46c0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"GET\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243]: Unexpected error:\n \u003c*errors.joinError | 0xc0012701e0\u003e: \n failed to cleanup resource group: failed listing hcp clusters in resourcegroup=\"basic-cluster-j2w6hk\": GET https://rp.j3045376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-j2w6hk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n errs: [\n \u003c*fmt.wrapError | 0xc0009f48c0\u003e{\n msg: \"failed to cleanup resource group: failed listing hcp clusters in resourcegroup=\\\"basic-cluster-j2w6hk\\\": GET https://rp.j3045376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-j2w6hk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0009f4860\u003e{\n msg: \"failed listing hcp clusters in resourcegroup=\\\"basic-cluster-j2w6hk\\\": GET https://rp.j3045376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-j2w6hk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001102540\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"107\"],\n \"X-Envoy-Upstream-Service-Time\": [\"2669\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"ff126f3d-baf1-4954-a9ec-cc35a45b7494\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 16:32:56 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0009f46c0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"GET\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"97b65f9e777bb09cab0bbcfe7e592e2f34cfcb87ae8d91b059fe97687f189bcf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4461/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033546562403045376","occurred_at":"2026-03-16T14:10:55Z","failed_at":"e2e","signature_id":"c396d9803ecea28ce0502d527a1fc52e959ed4316e762b2bdebc7deff942edb2","test_name":"Engineering should be able to retrieve expected metrics from the /metrics endpoint","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:74]: Failed to set up port-forward\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a026c0\u003e: \n port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.9:10250, code 500: 500 Internal Server Error\n {\n msg: \"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.9:10250, code 500: 500 Internal Server Error\",\n err: \u003c*errors.errorString | 0xc00094c780\u003e{\n s: \"error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.9:10250, code 500: 500 Internal Server Error\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:74]: Failed to set up port-forward\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a026c0\u003e: \n port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.9:10250, code 500: 500 Internal Server Error\n {\n msg: \"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.9:10250, code 500: 500 Internal Server Error\",\n err: \u003c*errors.errorString | 0xc00094c780\u003e{\n s: \"error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.9:10250, code 500: 500 Internal Server Error\",\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f27efe24b36721c3037189b0270b3c037f3dcd8e282db885731d7395d97e1e2a","failure_pattern":"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing \u003cip\u003e:\u003cport\u003e, code 500: 500 Internal Server Error","search_query":"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.9:10250, code 500: 500 Internal Server Error"}},{"row_id":"abe7122ee7e02df832f6b04efe8a3938cb6bd2573e2a8deb2cbecbc5724e8b5f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4461/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033546562403045376","occurred_at":"2026-03-16T14:10:55Z","failed_at":"e2e","signature_id":"cc01447c282423f130f110b0c546d43c434c86c437031c9cbb256bf909b6980a","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243]: Unexpected error:\n \u003c*errors.joinError | 0xc00156c168\u003e: \n failed to cleanup resource group: failed to detach NSGs from subnets in resource group external-auth-cluster-27vjn9: failed listing vnets in resource group external-auth-cluster-27vjn9: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-27vjn9/providers/Microsoft.Network/virtualNetworks\n --------------------------------------------------------------------------------\n RESPONSE 429: 429 Too Many Requests\n ERROR CODE: ResourceCollectionRequestsThrottled\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceCollectionRequestsThrottled\",\n \"message\": \"Operation 'Microsoft.Network/virtualNetworks/read' failed as server encountered too many requests. Please try after '40' seconds. Tracking Id is 'b2596673-aadc-4155-b729-25f1c0c9d169'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f08420\u003e{\n msg: \"failed to cleanup resource group: failed to detach NSGs from subnets in resource group external-auth-cluster-27vjn9: failed listing vnets in resource group external-auth-cluster-27vjn9: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-27vjn9/providers/Microsoft.Network/virtualNetworks\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: ResourceCollectionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"ResourceCollectionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Operation 'Microsoft.Network/virtualNetworks/read' failed as server encountered too many requests. Please try after '40' seconds. Tracking Id is 'b2596673-aadc-4155-b729-25f1c0c9d169'.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000f08400\u003e{\n msg: \"failed to detach NSGs from subnets in resource group external-auth-cluster-27vjn9: failed listing vnets in resource group external-auth-cluster-27vjn9: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-27vjn9/providers/Microsoft.Network/virtualNetworks\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: ResourceCollectionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"ResourceCollectionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Operation 'Microsoft.Network/virtualNetworks/read' failed as server encountered too many requests. Please try after '40' seconds. Tracking Id is 'b2596673-aadc-4155-b729-25f1c0c9d169'.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000f083c0\u003e{\n msg: \"failed listing vnets in resource group external-auth-cluster-27vjn9: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-27vjn9/providers/Microsoft.Network/virtualNetworks\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: ResourceCollectionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"ResourceCollectionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Operation 'Microsoft.Network/virtualNetworks/read' failed as server encountered too many requests. Please try after '40' seconds. Tracking Id is 'b2596673-aadc-4155-b729-25f1c0c9d169'.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000a88750\u003e{\n ErrorCode: \"ResourceCollectionRequestsThrottled\",\n StatusCode: 429,\n RawResponse: {\n Status: \"429 Too Many Requests\",\n StatusCode: 429,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Pragma\": [\"no-cache\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243]: Unexpected error:\n \u003c*errors.joinError | 0xc00156c168\u003e: \n failed to cleanup resource group: failed to detach NSGs from subnets in resource group external-auth-cluster-27vjn9: failed listing vnets in resource group external-auth-cluster-27vjn9: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-27vjn9/providers/Microsoft.Network/virtualNetworks\n --------------------------------------------------------------------------------\n RESPONSE 429: 429 Too Many Requests\n ERROR CODE: ResourceCollectionRequestsThrottled\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceCollectionRequestsThrottled\",\n \"message\": \"Operation 'Microsoft.Network/virtualNetworks/read' failed as server encountered too many requests. Please try after '40' seconds. Tracking Id is 'b2596673-aadc-4155-b729-25f1c0c9d169'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f08420\u003e{\n msg: \"failed to cleanup resource group: failed to detach NSGs from subnets in resource group external-auth-cluster-27vjn9: failed listing vnets in resource group external-auth-cluster-27vjn9: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-27vjn9/providers/Microsoft.Network/virtualNetworks\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: ResourceCollectionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"ResourceCollectionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Operation 'Microsoft.Network/virtualNetworks/read' failed as server encountered too many requests. Please try after '40' seconds. Tracking Id is 'b2596673-aadc-4155-b729-25f1c0c9d169'.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000f08400\u003e{\n msg: \"failed to detach NSGs from subnets in resource group external-auth-cluster-27vjn9: failed listing vnets in resource group external-auth-cluster-27vjn9: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-27vjn9/providers/Microsoft.Network/virtualNetworks\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: ResourceCollectionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"ResourceCollectionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Operation 'Microsoft.Network/virtualNetworks/read' failed as server encountered too many requests. Please try after '40' seconds. Tracking Id is 'b2596673-aadc-4155-b729-25f1c0c9d169'.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000f083c0\u003e{\n msg: \"failed listing vnets in resource group external-auth-cluster-27vjn9: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-27vjn9/providers/Microsoft.Network/virtualNetworks\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: ResourceCollectionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"ResourceCollectionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Operation 'Microsoft.Network/virtualNetworks/read' failed as server encountered too many requests. Please try after '40' seconds. Tracking Id is 'b2596673-aadc-4155-b729-25f1c0c9d169'.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000a88750\u003e{\n ErrorCode: \"ResourceCollectionRequestsThrottled\",\n StatusCode: 429,\n RawResponse: {\n Status: \"429 Too Many Requests\",\n StatusCode: 429,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Pragma\": [\"no-cache\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"27391593c89e1b6a3153b058ef31be166dd830240fb93e3caa733a62c02edc05","failure_pattern":"ERROR CODE: ResourceCollectionRequestsThrottled; provider Microsoft.Network","search_query":"ERROR CODE: ResourceCollectionRequestsThrottled"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6","27391593c89e1b6a3153b058ef31be166dd830240fb93e3caa733a62c02edc05","7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","f27efe24b36721c3037189b0270b3c037f3dcd8e282db885731d7395d97e1e2a"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/1955/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033533765682204672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":1955,"pr_state":"open","pr_sha":"f3b153a375111dc1f50de6fd382f772bcb7b7e9f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T13:19:59Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e44432c23ac4698c4f8a02716ec4273c1e62ed4479576034e24fa195c044fb21","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/1955/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033533765682204672","occurred_at":"2026-03-16T13:19:59Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/2312/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033533538963296256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":2312,"pr_state":"open","pr_sha":"37c1fdb9366a0b1198b571f8b2410d2a8ea54dfa","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T13:19:11Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"9416ef8ee9aa8192721f4b6dc30d03c2e7ccfa8594fde231716827466b1ccbe8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/2312/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033533538963296256","occurred_at":"2026-03-16T13:19:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/2422/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033533337129193472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":2422,"pr_state":"open","pr_sha":"88f665f6492c0951d302162ff744a54476a46608","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T13:18:23Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"0d49b1dc4791ba4b1e2943c57549f706088dc146241f9108daf9d210c89ebb2e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/2422/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033533337129193472","occurred_at":"2026-03-16T13:18:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/2593/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033533236864356352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":2593,"pr_state":"open","pr_sha":"c8ca820950115317a50a26f5b865161b54b6a2a8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T13:17:59Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"76bd8ad8a8f4860a937989b2bdc2e1e3ce1b501b93d061f163cce8b7a72b1ba7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/2593/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033533236864356352","occurred_at":"2026-03-16T13:17:59Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/2615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033533035789422592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":2615,"pr_state":"open","pr_sha":"6085e528a456bfa88ca048be3d032f87b12570a9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T13:17:11Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e1d91945d5f222ab4b6a0ca57f7f95e10721ab47f67bfebd632fe8f9865df140","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/2615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033533035789422592","occurred_at":"2026-03-16T13:17:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/2620/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033532854687764480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":2620,"pr_state":"open","pr_sha":"860d47cae4eec501f227268e7533125c94d3d04e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T13:16:23Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"27c1bd992f3be782aac4568cfad9cec31045fbabd63997d25499bcffd401a3a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/2620/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033532854687764480","occurred_at":"2026-03-16T13:16:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/2858/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033532532254838784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":2858,"pr_state":"open","pr_sha":"421c4022b5bc5d9f55f5b17f22f27df20678736a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T13:15:11Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"77b312631a3ad5bc1d48cbdd79fcfc9c4f15941a3736d43c6b25a75db24644e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/2858/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033532532254838784","occurred_at":"2026-03-16T13:15:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/2873/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033532331037298688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":2873,"pr_state":"open","pr_sha":"d7f12c97b5cc1bc6d0e58564e58973591487bfe8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T13:14:23Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"2ad0c458e7eb166ec426c7f5885e50b74f83905b6a9cd53822bddffe544e1b46","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/2873/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033532331037298688","occurred_at":"2026-03-16T13:14:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033532029831745536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3045,"pr_state":"open","pr_sha":"4a5b0be09ac27918e4f96895d3bdfeb952c80a82","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T13:13:11Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"bc3fce4669dc0b3838979b5c72d17ced55ce466f7cf733c250c11072c4066ec8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033532029831745536","occurred_at":"2026-03-16T13:13:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3061/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033531626117402624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3061,"pr_state":"open","pr_sha":"bed5752bc48223ba2044849fcd06be9823ba820e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T13:11:35Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"5445ea4686d42e6edc60150d00be0ba0405460dd080b58cac3b8508afbb3970a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3061/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033531626117402624","occurred_at":"2026-03-16T13:11:35Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3129/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033531223560687616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3129,"pr_state":"open","pr_sha":"815b1f320292a50711f12f50b777c02617cd7505","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T13:09:59Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"f98936e503995a278ef3f0553f3a14f07a90f4e6ebae22d2c303c0808cfca566","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3129/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033531223560687616","occurred_at":"2026-03-16T13:09:59Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033531022578028544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3172,"pr_state":"closed","pr_sha":"0e2cea398f94dcc7cbca89c30addf4b3493e9dfa","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T13:09:11Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"76dea52673313fa03465fd4b18886ea74b15a62ad26e4b2762f75d17aa08362f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033531022578028544","occurred_at":"2026-03-16T13:09:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033530955733405696","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4117,"pr_state":"closed","pr_sha":"056231ac183a0725b7aa46a7473a4b2c4bf04aca","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T13:08:44Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"f9a9677bd84c92c3a753234219fee7ad34824f7b823c9a96d9c820c593d2d34d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033530955733405696","occurred_at":"2026-03-16T13:08:44Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3243/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033530619866124288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3243,"pr_state":"open","pr_sha":"c682e6ca1bf44a701a96892696e8c80fe5a0dc90","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T13:07:35Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b32001e0ef96fdb060f5e16b27bd6fe21ffb8b704f3c3ae308b85f319b57d1e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3243/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033530619866124288","occurred_at":"2026-03-16T13:07:35Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3390/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033530418128490496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3390,"pr_state":"open","pr_sha":"4bed8a147e69cbf9de2470d1364d4c636941604b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T13:06:47Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"a03331431dc3cbdfd8645b4de1e2ea4e1c9bf3db7c8e1c0d8548c0f266c0542d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3390/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033530418128490496","occurred_at":"2026-03-16T13:06:47Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033530215988203520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3414,"pr_state":"open","pr_sha":"217ba23ec66f6517ba41f11ab8f49ed68a6231fb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T13:05:59Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"46a589111ecfec40ccf26866dd4c60384940d8fc7e34c42660df1a9260c30b76","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033530215988203520","occurred_at":"2026-03-16T13:05:59Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033530115253604352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3557,"pr_state":"open","pr_sha":"c58bad3dbc7f4e393442cdcfcf4a24afc24fc025","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T13:05:35Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"adb7a8e1fc80971133a08b3dd6610dcfbaa1d518d693d81ef46527fda9480a44","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033530115253604352","occurred_at":"2026-03-16T13:05:35Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3564/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033530014628057088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3564,"pr_state":"open","pr_sha":"958343460861eb54ae15d5ecc1047847ed0808a3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T13:05:11Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"8e0790e727914e86f862bcd459f8900e69ea96507a9bb00ea632eddc86dfc61d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3564/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033530014628057088","occurred_at":"2026-03-16T13:05:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033529914786844672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3585,"pr_state":"open","pr_sha":"c5298fcb330459f51f90849c96419d8086080ea4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T13:04:47Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"87902f7bf4a32238f6b0a422438298dc02dd2074c57b4c1591fe1b9fd414a48d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033529914786844672","occurred_at":"2026-03-16T13:04:47Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033529804740890624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4403,"pr_state":"open","pr_sha":"2be65f983a6ddc8944ee1217ce1a91422e6c24dc","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T13:04:21Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"7637032d6d401e05191262dfd022f43de75ca692857e074a669290a90a750291","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033529804740890624","occurred_at":"2026-03-16T13:04:21Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3596/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033529715033116672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3596,"pr_state":"open","pr_sha":"fc77837abe7ce5e9f3d81e9c0307aaefa902b303","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T13:03:59Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"66ddf032ffe8dfe0a6d265bf6b14995cccdd40057898557251f820418fc721ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3596/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033529715033116672","occurred_at":"2026-03-16T13:03:59Z","failed_at":"provision","signature_id":"5a16efce6856cb7ec3c289414cb5c6df861b61434d949293d8a83736a8fb478f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-16T15:38:43.499Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Shell\\n Command: make deploy\\n\"\ntime=2026-03-16T15:40:41.168Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Shell Step, failed to execute shell command: make[2]: Entering directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\nkubectl create namespace clusters-service --dry-run=client -o json | kubectl apply -f - \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_CLIENT_ID=0abd6cee-971e-4831-bf65-52563f64495b \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_TENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nkubectl label namespace clusters-service \\\"istio.io/rev=prod-stable\\\" --overwrite=true \u0026\u0026 \\\\\\nAZURE_CS_MI_CLIENT_ID=838dbe80-d244-413a-9241-6ad2025a235b \u0026\u0026 \\\\\\nTENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nOIDC_BLOB_SERVICE_ENDPOINT=https://arohcpprowoidcj3116672.blob.core.windows.net/ \u0026\u0026 \\\\\\nOIDC_ISSUER_BASE_ENDPOINT=https://arohcpprowoidcj3116672.z1.web.core.windows.net/ \u0026\u0026 \\\\\\nDB_HOST=$(if [ \\\"false\\\" = \\\"true\\\" ]; then az postgres flexible-server show -g hcp-underlay-prow-j3116672 -n arohcp-prow-dbcs-j3116672 --query fullyQualifiedDomainName -o tsv; else echo \\\"ocm-cs-db\\\"; fi) \u0026\u0026 \\\\\\nOVERRIDES=$(if [ \\\"false\\\" = \\\"true\\\" ]; then echo \\\"azuredb.values.yaml\\\"; else echo \\\"containerdb.values.yaml\\\"; fi) \u0026\u0026 \\\\\\nZONE_RESOURCE_ID=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j3116672/providers/Microsoft.Network/dnszones/j3116672.hcp.osadev.cloud \u0026\u0026 \\\\\\nCX_SECRETS_KV_URL=\\\"https://ah-prow-cx-j3116672-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\nCX_MI_KV_URL=\\\"https://ah-prow-mi-j3116672-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\n./hack/helm.sh cluster-service helm-charts/cluster-service clusters-service \\\\\\n -f helm-charts/cluster-service/${OVERRIDES} \\\\\\n --set serviceAccountName=clusters-service \\\\\\n --set environment=arohcpprow \\\\\\n --set azureCsMiClientId=${AZURE_CS_MI_CLIENT_ID} \\\\\\n --set oidcIssuerBlobServiceUrl=${OIDC_BLOB_SERVICE_ENDPOINT} \\\\\\n --set oidcIssuerBaseUrl=${OIDC_ISSUER_BASE_ENDPOINT} \\\\\\n --set tlsCertificatesIssuer=${TLS_CERTIFICATES_ISSUER} \\\\\\n --set tlsCertificatesEnabled=true \\\\\\n --set denyAssignments=disabled \\\\\\n --set tenantId=${TENANT_ID} \\\\\\n --set region=westus3 \\\\\\n --set serviceKeyvaultName=aro-hcp-dev-svc-kv \\\\\\n --set replicas=3 \\\\\\n --set imageRegistry=arohcpsvcdev.azurecr.io \\\\\\n --set imageRepository=app-sre/aro-hcp-clusters-service \\\\\\n --set imageDigest=sha256:6ec7ecb11515ea6f8be560c4c043b518ef5c5ccc2a0aaafb2a45ee79c25a9d80 \\\\\\n --set global.imageDigest=sha256:6ec7ecb11515ea6f8be560c4c043b518ef5c5ccc2a0aaafb2a45ee79c25a9d80 \\\\\\n --set azureFirstPartyApplicationClientId=b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 \\\\\\n --set batchProcessesDryRun=true \\\\\\n --set batchProcesses=\\\"\\\" \\\\\\n --set fpaCertName=firstPartyCert2 \\\\\\n --set ocpAcrResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev \\\\\\n --set ocpAcrUrl=arohcpocpdev.azurecr.io \\\\\\n --set shard.consumerName=\\\"hcp-underlay-j3116672-mgmt-1\\\" \\\\\\n --set shard.zoneResourceId=\\\"${ZONE_RESOURCE_ID}\\\" \\\\\\n --set shard.cxSecretsKeyVaultUrl=\\\"${CX_SECRETS_KV_URL}\\\" \\\\\\n --set shard.cxMiKeyVaultUrl=\\\"${CX_MI_KV_URL}\\\" \\\\\\n --set shard.cxSecretsKeyVaultMiClientId=\\\"232f5520-5f28-4117-986c-3714aa1e0fc7\\\" \\\\\\n --set shard.maestroRestUrl=\\\"http://maestro.maestro.svc.cluster.local:8000\\\" \\\\\\n --set shard.maestroGrpUrl=\\\"maestro-grpc.maestro.svc.cluster.local:8090\\\" \\\\\\n --set shard.Id=$(python3 -c \\\"import uuid; print(uuid.uuid5(uuid.UUID('916f9976-e1c0-4afd-b84c-5d5c94fbeaed'), 'prow-j3116672-mgmt-1'))\\\") \\\\\\n --set shard.aksManagementClusterResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j3116672-mgmt-1/providers/Microsoft.ContainerService/managedClusters/prow-j3116672-mgmt-1 \\\\\\n --set databaseHost=${DB_HOST} \\\\\\n --set azureMiMockServicePrincipalPrincipalId=d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 \\\\\\n --set azureMiMockServicePrincipalClientId=e8723db7-9b9e-46a4-9f7d-64d75c3534f0 \\\\\\n --set azureMiMockServicePrincipalCertName=msiMockCert2 \\\\\\n --set azureArmHelperIdentityCertName=armHelperCert2 \\\\\\n --set azureArmHelperIdentityClientId=3331e670-0804-48e8-a086-6241671ddc93 \\\\\\n --set azureArmHelperMockFpaPrincipalId=47f69502-0065-4d9a-b19b-d403e183d2f4 \\\\\\n --set azureCloudEnvironmentName=AzurePublicCloud \\\\\\n --set pullBinding.workloadIdentityClientId=\\\"${IMAGE_PULLER_MI_CLIENT_ID}\\\" \\\\\\n --set pullBinding.workloadIdentityTenantId=\\\"${IMAGE_PULLER_MI_TENANT_ID}\\\" \\\\\\n --set pullBinding.registry=arohcpsvcdev.azurecr.io \\\\\\n --set pullBinding.scope=repository:app-sre/aro-hcp-clusters-service:pull \\\\\\n --set managedIdentitiesDataPlaneAudienceResource=https://dummy.org \\\\\\n --set tracing.address=http://ingest.observability:4318 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxSurge=1 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxUnavailable=0 \\\\\\n --set provisionShardClusterLimit=100 \\\\\\n --set ocpVersions.defaultVersion.version=4.20.8 \\\\\\n --set ocpVersions.channelGroups.stable.minVersion=4.19.0 \\\\\\n --set ocpVersions.channelGroups.stable.maxVersion= \\\\\\n --set ocpVersions.channelGroups.candidate.minVersion=4.19.7 \\\\\\n --set ocpVersions.channelGroups.candidate.maxVersion= \\\\\\n --set ocpVersions.channelGroups.nightly.minVersion=4.19.0-0.nightly-20200101 \\\\\\n --set ocpVersions.channelGroups.nightly.maxVersion= \\\\\\n --set azureOperatorsMI.roleSetName=dev \\\\\\n --set deployment.zoneCount=4\\nnamespace/clusters-service created\\nnamespace/clusters-service labeled\\nRun Helm upgrade with release name cluster-service in namespace clusters-service\\nRelease \\\"cluster-service\\\" does not exist. Installing it now.\\nError: Unauthorized\\nmake[2]: *** [Makefile:32: deploy] Error 1\\nmake[2]: Leaving directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\n exit status 2\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40","failure_pattern":"make[2]: *** [Makefile:32: deploy] Error 1","search_query":"make[2]: *** [Makefile:32: deploy] Error 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033529511475154944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3661,"pr_state":"closed","pr_sha":"6029601d96d68c2ea706cbf5922d5348b46bc432","final_merged_sha":"6029601d96d68c2ea706cbf5922d5348b46bc432","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T13:03:11Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"49e93b409084794ea32b86c3e00bfdea71b7d094193ea75cd13c2535744e68e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033529511475154944","occurred_at":"2026-03-16T13:03:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3680/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033529158994235392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3680,"pr_state":"closed","pr_sha":"7c52cdfc9d9a15419e56b1d62980ec306fdfd23f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T13:01:47Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ca6110d641b681163d695f373aea9284c49037bb3826b5225e973a02391bb61f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3680/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033529158994235392","occurred_at":"2026-03-16T13:01:47Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3696/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033528307676352512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3696,"pr_state":"open","pr_sha":"39008467af52385dd11f0ef4cc894e5bc149fab8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:58:23Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"8fd29c07e0d202a92657947f6383bb75f5020c9de4c8f7f64e2efbcba6cf312d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3696/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033528307676352512","occurred_at":"2026-03-16T12:58:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3704/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033528051656036352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3704,"pr_state":"closed","pr_sha":"2b9a7c8c45ae714328a4d9cdf7e36fc1167f96a2","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:57:23Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"95ed8c36dfc9168580d34b1a9b8d220c486cc1a99cfc67c26f63df86b712e7ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3704/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033528051656036352","occurred_at":"2026-03-16T12:57:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3729/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033527750559535104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3729,"pr_state":"open","pr_sha":"a81d500b4496b42bc3a357fce94fe95e5fe5664b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:56:11Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"c12a86c5145e94a501511fb50762f76b11682922372a63f20239f1ee6663d517","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3729/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033527750559535104","occurred_at":"2026-03-16T12:56:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3760/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033527549367160832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3760,"pr_state":"open","pr_sha":"12f8f995c2909a2f0abe8184017fe37fe172e5c0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:55:23Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"c5f0a00bbc1979f447179ca216601d69283217bbc2af0cae66b87959fe73790a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3760/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033527549367160832","occurred_at":"2026-03-16T12:55:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3779/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033527475123785728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3779,"pr_state":"open","pr_sha":"15a2d96d309f75bbdf732dec6ff07752bbe4fbab","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:54:35Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"57a0151d16be68f880bfc129181182548db52b3d060abec9873ab99d8f6187c6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3779/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033527475123785728","occurred_at":"2026-03-16T12:54:35Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3794/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033527018653487104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3794,"pr_state":"open","pr_sha":"704ba8648c496ba0af9fc250f5b082d27a843048","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:53:11Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"a7389d83bb94a3e7c17992d3bac09a64ef247b8d9d22f0b853dc47d99729d0dd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3794/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033527018653487104","occurred_at":"2026-03-16T12:53:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3795/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033526893981995008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3795,"pr_state":"open","pr_sha":"51248eac2cbff1a1e8cc5b609fb1a2c943e2f08f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:52:47Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"01689991730f58dd58f8b75c63c6c3531e63496fb6e38a16a1dc18ffc478975b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3795/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033526893981995008","occurred_at":"2026-03-16T12:52:47Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033526794207891456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3823,"pr_state":"closed","pr_sha":"d91199355a8d2d1e2888e3ddb9de70bf4bff9444","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:52:23Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"49b0c41353a8ebedb0af4a50e7621d42cc993c026515c76f8eae4f286da84c28","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033526794207891456","occurred_at":"2026-03-16T12:52:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3827/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033526592944214016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3827,"pr_state":"open","pr_sha":"4c4a64476940ea80644c95d447100540814969e5","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:51:35Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"f322fca6429c139a2aef3da3068fb178b0695faf4106770d472986ad75af5b29","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3827/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033526592944214016","occurred_at":"2026-03-16T12:51:35Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3839/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033526409397276672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3839,"pr_state":"open","pr_sha":"c90f337db11e25bac6ac80dab90106b5957482e5","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:50:47Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"221909a675f59618c73af82c67094d2a2634905d4f6b2f903f4900abc7fa2fc7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3839/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033526409397276672","occurred_at":"2026-03-16T12:50:47Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3840/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033526190228115456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3840,"pr_state":"open","pr_sha":"bde613f77582172fd0c5419246a0aba6eeadc104","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:49:59Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"41da26704d3a2b9035d927251867d6abff48828b484493eeec9b14be5e2e3b23","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3840/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033526190228115456","occurred_at":"2026-03-16T12:49:59Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3841/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033525989014769664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3841,"pr_state":"open","pr_sha":"0a38030f3b7e3595f9b1762c4b260e227574d1f4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:49:11Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b8b59895cbce7c095ffe79cde43fbcb0c9b1b5681494cd9f2b658fc659e84f07","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3841/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033525989014769664","occurred_at":"2026-03-16T12:49:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3860/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033525284233285632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3860,"pr_state":"open","pr_sha":"bbd70490488aaaecb37d3fdcb6380f9d83ee4697","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:46:23Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"67a6fe69b21bbe5f66ba80f9a3baf855324693f3221b44ce9eff7f0a158fba91","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3860/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033525284233285632","occurred_at":"2026-03-16T12:46:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033524680312229888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3880,"pr_state":"open","pr_sha":"7b88114939e6d13d5868f702667bcfcac8d7c2da","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:43:59Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"61c6cc23d90cd2597e7afe45ce107bf49a30a8502b37d856a6d103e801e7d14e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033524680312229888","occurred_at":"2026-03-16T12:43:59Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3907/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033523751315836928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3907,"pr_state":"closed","pr_sha":"afc342563502c45ec5834534bfc50b2983ff77b0","final_merged_sha":"e9ca15da6d935a378fb165189c2441a771314f73","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:40:11Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ee13dc418d84957782a11cffd8f099202d8642f5fe7b31072e56dee46515b094","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3907/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033523751315836928","occurred_at":"2026-03-16T12:40:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3913/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033523532234756096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3913,"pr_state":"open","pr_sha":"2841bf78b2a3642eec1fe39e1db0fc79dbb181fb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:39:23Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"58624cee06a4fb9526d07be08fdd818ee85076c09bf1f475871d6a593c6f285e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3913/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033523532234756096","occurred_at":"2026-03-16T12:39:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3921/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033523321596809216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3921,"pr_state":"open","pr_sha":"babd84cb7e174064b966b744e5365c30776fc848","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:38:35Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b6976e1a8ef524f1f5c9db4585d78e9b7a0afeb45818244f8584c47ba500846e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3921/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033523321596809216","occurred_at":"2026-03-16T12:38:35Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033523119192281088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3925,"pr_state":"open","pr_sha":"ff393c2d23195248d6365848c370f7dee26db58e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:37:47Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"fd14c55cf391716cd50ab731402a8ebd5b066dc38744ff61840eb0fab8cb7c29","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033523119192281088","occurred_at":"2026-03-16T12:37:47Z","failed_at":"provision","signature_id":"14e1139051b92a0259280ab4bba3ccc679edd3541b36e2204a8d4ec84f155427","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/prometheus","test_suite":"step graph","failure_text":"time=2026-03-16T15:11:09.550Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-03-16T15:11:10.202Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:d7e8257f8d8bce64b6df55f81fba92011a6a77269b3350f8b997b152af348dba]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:414103ed-9e0e-48b0-af39-d5334fe1b53c]] prometheusSpec:map[externalLabels:map[cluster:prow-j2281088-svc] hcpRemoteWriteUrl:NONE image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:dca4b35f73046dc4803c0160c2e6be5a1736a716b26ef35590864d36dfdf950a] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j2281088-svc-yeyq.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-3ea31b0dcf98492e9a18d301fe28c4d7/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-16T15:11:10.202Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-16T15:11:10.846Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace prometheus: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ecf60ba8bdd57b40e995d27f45ca5a2fb0a8082c735c55d75d0af026c1021bb","failure_pattern":"failed to apply namespace prometheus: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace prometheus: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["8ecf60ba8bdd57b40e995d27f45ca5a2fb0a8082c735c55d75d0af026c1021bb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033523019405594624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3952,"pr_state":"closed","pr_sha":"baff2916dc32c2f9eb4df2cd19e2f680f65af352","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:37:23Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"cc23b947a15d7f6c0a68f184914cea3dda4131118b157c1ff7ca1cf882fb0ead","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033523019405594624","occurred_at":"2026-03-16T12:37:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3957/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033522818146111488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3957,"pr_state":"open","pr_sha":"23fa46c1c76e2f247ca59cd5b097ae9508c253eb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:36:35Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"2f920520a95b5a2c55d3abb3279e7bb873b80e3c72cf544ef32bcaf3477e4162","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3957/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033522818146111488","occurred_at":"2026-03-16T12:36:35Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3960/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033522615754166272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3960,"pr_state":"closed","pr_sha":"b65c3a633c23914dd7243b1a0245bb0be24e2387","final_merged_sha":"b65c3a633c23914dd7243b1a0245bb0be24e2387","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T12:35:47Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"c6784ee45ce263df72bb78625790d9c806b22ce1b4bd84bc1668153b7e4d4c0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3960/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033522615754166272","occurred_at":"2026-03-16T12:35:47Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3973/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033522263596208128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3973,"pr_state":"closed","pr_sha":"fee956dd6a80e69211f67537c45bd0a8b4119ea1","final_merged_sha":"445d308e2fc4c51ba6b9b76df86f35addde8a91b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:34:23Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"a72a4d298c96844a1ad2e7ce66b835cda151b940c43a8b6bf78e2198fce28eb3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3973/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033522263596208128","occurred_at":"2026-03-16T12:34:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3975/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033522164036014080","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3975,"pr_state":"open","pr_sha":"dfad7bfde8cec914450623922a9111cc8556d8a4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:33:59Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"a80bb888a281b9a289c3042c62092430d723a23f212517684005d751682008e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3975/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033522164036014080","occurred_at":"2026-03-16T12:33:59Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033521962612953088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3977,"pr_state":"closed","pr_sha":"a61671ec8db1d861fd7fa8a89e4db260fa048818","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:33:11Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"0e82ecdcd02d8f671de6ce9c3f0443f410f90db19243631f5b11b4cb68d11d8c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033521962612953088","occurred_at":"2026-03-16T12:33:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033521509728784384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4004,"pr_state":"closed","pr_sha":"be9daeda83ecf50c0b263592b75cf782417142e9","final_merged_sha":"9feee09ab6996cfe7dd433b262345799b7851593","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:31:23Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"79c61887d62356a7e6a17954cd8e93f61a5e0efe81edb9802431640b6cf72632","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033521509728784384","occurred_at":"2026-03-16T12:31:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4040/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033521308288946176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4040,"pr_state":"open","pr_sha":"9e94925f147f55421a2560c9da722ba52bed0e5f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:30:35Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"629af88dc2828bea462f0129f31c027b3d4194c6bdf5e8e89eb403bc242bd859","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4040/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033521308288946176","occurred_at":"2026-03-16T12:30:35Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033521110032584704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4044,"pr_state":"closed","pr_sha":"f5b50ce8ef55499ee0e32bf353c211688e60152c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:29:47Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"43cdfaaff656e307151456b39f2eecb544e9533cd18263c54ebcaa87e78aec29","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033521110032584704","occurred_at":"2026-03-16T12:29:47Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033520934219943936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4045,"pr_state":"open","pr_sha":"3207545fca598a0bbc926207da9f3ebd12d86873","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:28:59Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"3fcbc14767a632ee5d3a529d9bfb526f3759f832aed4de1005ad07bebdecbb2d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033520934219943936","occurred_at":"2026-03-16T12:28:59Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4449/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033520858718277632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4449,"pr_state":"closed","pr_sha":"3b95084d22c390ee56e53ed8653eb39c2499db41","final_merged_sha":"205984e6ef0d1b76b1e624468408854ec2a65a8c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:28:30Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b21923e3a170f3d18c6e04e6f95c24fc0f547baab8e092cc348e164be5e544ad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4449/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033520858718277632","occurred_at":"2026-03-16T12:28:30Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4046/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033520816779431936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4046,"pr_state":"open","pr_sha":"456c77ebac1632bd0f5ac4eec9ad5eeca97f0ffb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:28:11Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"1067b4a6108ad2a4d2a18b7267d2eb0b7bd6caab33ea8c95c8d399d5d0a70ca7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4046/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033520816779431936","occurred_at":"2026-03-16T12:28:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4069/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033520502831583232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4069,"pr_state":"closed","pr_sha":"0185c7ce0f12ca7e96da04effddb02a63c5b0c54","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:27:23Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"c2a1749fc7d04139bc9f58149c4a2c01f07102b92799a67319c32ed8ab7c3754","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4069/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033520502831583232","occurred_at":"2026-03-16T12:27:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4084/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033520049817391104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4084,"pr_state":"closed","pr_sha":"a35acbe01efc2addde679a45c4cf72d4d1038fc5","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:25:35Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b0ad089f78d4291ff17a9f6184903ad569dd2f7c2dcca86876c0c6c0d2ce8a69","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4084/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033520049817391104","occurred_at":"2026-03-16T12:25:35Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4095/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033519848457244672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4095,"pr_state":"open","pr_sha":"8f711fab81a8d9300197820c371560b67cc2acd0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:24:47Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b2dfb9f6502ee301074f5e2ff8e883a6fac930ae895b93b9d86368557a952f3f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4095/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033519848457244672","occurred_at":"2026-03-16T12:24:47Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033519445715980288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4117,"pr_state":"closed","pr_sha":"2eda04857baa3d253d3cc30afc7eafe330c6b6b2","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:23:11Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"15c804a9e1f77ea186e8cce29e56592c228ae56a9d56993a90bb6d94c76c31f0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033519445715980288","occurred_at":"2026-03-16T12:23:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033519244355833856","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4133,"pr_state":"closed","pr_sha":"1c913457687a8c291cae0bfd349b55fbe571f0f3","final_merged_sha":"4f10ef55a408ea39a4f644c08d4ab2d01cf39c2a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:22:23Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"2748d1140ecdb9eaecd76e00fcfa0362789d8d22e1bcc5ec3bba5cd97d5a7a93","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033519244355833856","occurred_at":"2026-03-16T12:22:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4137/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033519042274267136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4137,"pr_state":"closed","pr_sha":"c004a2c9d60d11830b4655b33cc48cd47290f025","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:21:35Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"18e56287e673b269faf2f1bb5d04869a8d871f973db93a5c6b0eef3f193dc3aa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4137/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033519042274267136","occurred_at":"2026-03-16T12:21:35Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4138/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033518946325368832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4138,"pr_state":"closed","pr_sha":"c179b3a8ca7caed0b9ef6bfac44db090f2575364","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:21:11Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b8944976f40cc1417a86e79838292fadf2c4d7aaf72896626213a2e98a3ae7d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4138/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033518946325368832","occurred_at":"2026-03-16T12:21:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033518741291012096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4157,"pr_state":"open","pr_sha":"730aedc1e2f03e62758aaba9b43c4ccdd249c551","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:20:23Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"fc60121a1952f66669fddd7b89fe3337e9435dd3deb5116b028e55dd73cd70dd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033518741291012096","occurred_at":"2026-03-16T12:20:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4161/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033518539817619456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4161,"pr_state":"closed","pr_sha":"661d925a59c33e9c0177a3c2b6c751a967f69b02","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:19:35Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"edc48514b0009805b355719781fe3d63553b9929d989bca702969f00bbc3f044","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4161/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033518539817619456","occurred_at":"2026-03-16T12:19:35Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4171/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033518337689915392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4171,"pr_state":"closed","pr_sha":"6881981dc7284f4f636c5c7c4def8017b056604d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:18:47Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"0be610eb95ee7ce4533ee8bb841d9453c6b959fd4a5af82f0a7eeb5fbb9e8597","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4171/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033518337689915392","occurred_at":"2026-03-16T12:18:47Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033518238310076416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4185,"pr_state":"closed","pr_sha":"97b8851ea5195c5fd51cb1905f7c8a9c8f36ff37","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:18:23Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"62329ba0971819e11ee27df19a06cee398a0e10c6d5fba7b48da8f876a496981","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033518238310076416","occurred_at":"2026-03-16T12:18:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4188/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033518036664717312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4188,"pr_state":"closed","pr_sha":"c45e5a90356f27c66b7010a7081be7a8eb034202","final_merged_sha":"ad7003e5a941946bb5a1fed27c539b6ee3d95872","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:17:35Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"3731586d4ba9c518657833323142cee260759dd003d7ad579eae6d86362f8144","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4188/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033518036664717312","occurred_at":"2026-03-16T12:17:35Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4189/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033517864241074176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4189,"pr_state":"closed","pr_sha":"cbb5a982644bae16afb394f00697943175bf9496","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:16:47Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"96f0b3000415f7fbe6860e51ef4329ff6e56be902a0aaa7d4aa2594ecae04047","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4189/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033517864241074176","occurred_at":"2026-03-16T12:16:47Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4192/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033517634070253568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4192,"pr_state":"open","pr_sha":"83e208ab20c46816a256f852c17b5d66c9dd79f9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:15:59Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"6c1df8eaaced5f10680f2ae12b54dec6d3e654d290a2287b64c86361c619fc90","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4192/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033517634070253568","occurred_at":"2026-03-16T12:15:59Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4193/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033517433213423616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4193,"pr_state":"open","pr_sha":"6bb6adae75a297b794da681e782deb2aedf27e95","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:15:11Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"103424d2d85200fc8344117edd42f3cff609518e78264d7aba228b4914d44ed6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4193/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033517433213423616","occurred_at":"2026-03-16T12:15:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033517231354155008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4198,"pr_state":"open","pr_sha":"1b8d093b002bc026fd375fd63f1983eb3a0852e1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:14:23Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"4ff35b21afe155715ed3fc19da6c14d67881db78ef9505b04c62712bd0267b51","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033517231354155008","occurred_at":"2026-03-16T12:14:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4210/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033516928277942272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4210,"pr_state":"open","pr_sha":"bafae7a884337d3fd1d3147626c6999a8d28d04d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:13:11Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"54c00647808b61c809019c4f5db109a88078fa8ccf4b42ab6ca78babdca2d766","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4210/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033516928277942272","occurred_at":"2026-03-16T12:13:11Z","failed_at":"e2e","signature_id":"0c5f8deeb55b9d25c37b5c8ca7a605c8160dde110f34edc193c46dcc1910b946","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0015727f0\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0015727f0\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}},{"row_id":"86ead88fbf78be3ac38ec63abaecc1e3669b261efab6b4511153ba2035329c1e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4210/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033516928277942272","occurred_at":"2026-03-16T12:13:11Z","failed_at":"e2e","signature_id":"f74798b2075b8656a28bac6173b663d12ae0243abe59d9e2557e35ed82acabf7","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001546b80\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-kpmzkj\" to finish creating: GET https://rp.j7942272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a32d5f3-83b0-45cf-974c-a52cbc080006\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a32d5f3-83b0-45cf-974c-a52cbc080006\",\n \"name\": \"8a32d5f3-83b0-45cf-974c-a52cbc080006\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:54:48.303502861Z\",\n \"endTime\": \"2026-03-16T14:55:48.444999538Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-150/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-150/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-kpmzkj\\\" to finish creating: GET https://rp.j7942272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a32d5f3-83b0-45cf-974c-a52cbc080006\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a32d5f3-83b0-45cf-974c-a52cbc080006\\\",\\n \\\"name\\\": \\\"8a32d5f3-83b0-45cf-974c-a52cbc080006\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:54:48.303502861Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:55:48.444999538Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-150/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-150/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001546b60\u003e{\n msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-kpmzkj\\\" to finish creating: GET https://rp.j7942272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a32d5f3-83b0-45cf-974c-a52cbc080006\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a32d5f3-83b0-45cf-974c-a52cbc080006\\\",\\n \\\"name\\\": \\\"8a32d5f3-83b0-45cf-974c-a52cbc080006\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:54:48.303502861Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:55:48.444999538Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-150/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-150/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001546b80\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-kpmzkj\" to finish creating: GET https://rp.j7942272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a32d5f3-83b0-45cf-974c-a52cbc080006\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a32d5f3-83b0-45cf-974c-a52cbc080006\",\n \"name\": \"8a32d5f3-83b0-45cf-974c-a52cbc080006\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:54:48.303502861Z\",\n \"endTime\": \"2026-03-16T14:55:48.444999538Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-150/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-150/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-kpmzkj\\\" to finish creating: GET https://rp.j7942272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a32d5f3-83b0-45cf-974c-a52cbc080006\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a32d5f3-83b0-45cf-974c-a52cbc080006\\\",\\n \\\"name\\\": \\\"8a32d5f3-83b0-45cf-974c-a52cbc080006\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:54:48.303502861Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:55:48.444999538Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-150/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-150/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001546b60\u003e{\n msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-kpmzkj\\\" to finish creating: GET https://rp.j7942272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a32d5f3-83b0-45cf-974c-a52cbc080006\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a32d5f3-83b0-45cf-974c-a52cbc080006\\\",\\n \\\"name\\\": \\\"8a32d5f3-83b0-45cf-974c-a52cbc080006\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:54:48.303502861Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:55:48.444999538Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-150/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-150/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e3f288378bd9567a2f3e343610ad2f03c0ca801c1edcec18dfc250fdc0725144","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4210/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033516928277942272","occurred_at":"2026-03-16T12:13:11Z","failed_at":"e2e","signature_id":"ca87ca6bc70528a003bad2948f884cba69f2e580676b61a2e42ee8a69de67c48","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001540120\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-f5njwd\" to finish creating: GET https://rp.j7942272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01e17f5d-e2ff-4f99-9918-11e18c2490ab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01e17f5d-e2ff-4f99-9918-11e18c2490ab\",\n \"name\": \"01e17f5d-e2ff-4f99-9918-11e18c2490ab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:54:56.798653664Z\",\n \"endTime\": \"2026-03-16T14:55:38.486152327Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-140/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-140/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-f5njwd\\\" to finish creating: GET https://rp.j7942272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01e17f5d-e2ff-4f99-9918-11e18c2490ab\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01e17f5d-e2ff-4f99-9918-11e18c2490ab\\\",\\n \\\"name\\\": \\\"01e17f5d-e2ff-4f99-9918-11e18c2490ab\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:54:56.798653664Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:55:38.486152327Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-140/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-140/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001540100\u003e{\n msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-f5njwd\\\" to finish creating: GET https://rp.j7942272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01e17f5d-e2ff-4f99-9918-11e18c2490ab\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01e17f5d-e2ff-4f99-9918-11e18c2490ab\\\",\\n \\\"name\\\": \\\"01e17f5d-e2ff-4f99-9918-11e18c2490ab\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:54:56.798653664Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:55:38.486152327Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-140/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-140/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionReque...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001540120\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-f5njwd\" to finish creating: GET https://rp.j7942272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01e17f5d-e2ff-4f99-9918-11e18c2490ab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01e17f5d-e2ff-4f99-9918-11e18c2490ab\",\n \"name\": \"01e17f5d-e2ff-4f99-9918-11e18c2490ab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:54:56.798653664Z\",\n \"endTime\": \"2026-03-16T14:55:38.486152327Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-140/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-140/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-f5njwd\\\" to finish creating: GET https://rp.j7942272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01e17f5d-e2ff-4f99-9918-11e18c2490ab\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01e17f5d-e2ff-4f99-9918-11e18c2490ab\\\",\\n \\\"name\\\": \\\"01e17f5d-e2ff-4f99-9918-11e18c2490ab\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:54:56.798653664Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:55:38.486152327Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-140/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-140/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001540100\u003e{\n msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-f5njwd\\\" to finish creating: GET https://rp.j7942272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01e17f5d-e2ff-4f99-9918-11e18c2490ab\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01e17f5d-e2ff-4f99-9918-11e18c2490ab\\\",\\n \\\"name\\\": \\\"01e17f5d-e2ff-4f99-9918-11e18c2490ab\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:54:56.798653664Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:55:38.486152327Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-140/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-140/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionReque...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ec569c678026822bd0dbc6bbfb3c96c3561af1fc27f87696e564db87421eea07","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4210/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033516928277942272","occurred_at":"2026-03-16T12:13:11Z","failed_at":"e2e","signature_id":"a22b77c4ea65bf871cca664fbff9e01f515c63ad144b05d74423bfde44bc3fdd","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000f407e0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-kmh24k\" to finish creating: GET https://rp.j7942272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fcc9c987-dc72-47a5-acd2-7492bcbbcd08\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fcc9c987-dc72-47a5-acd2-7492bcbbcd08\",\n \"name\": \"fcc9c987-dc72-47a5-acd2-7492bcbbcd08\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:54:48.473384961Z\",\n \"endTime\": \"2026-03-16T14:56:32.581499648Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-202/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-202/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster ea-cluster: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-kmh24k\\\" to finish creating: GET https://rp.j7942272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fcc9c987-dc72-47a5-acd2-7492bcbbcd08\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fcc9c987-dc72-47a5-acd2-7492bcbbcd08\\\",\\n \\\"name\\\": \\\"fcc9c987-dc72-47a5-acd2-7492bcbbcd08\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:54:48.473384961Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:56:32.581499648Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-202/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-202/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000f407c0\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-kmh24k\\\" to finish creating: GET https://rp.j7942272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fcc9c987-dc72-47a5-acd2-7492bcbbcd08\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fcc9c987-dc72-47a5-acd2-7492bcbbcd08\\\",\\n \\\"name\\\": \\\"fcc9c987-dc72-47a5-acd2-7492bcbbcd08\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:54:48.473384961Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:56:32.581499648Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-202/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-202/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n----------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000f407e0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-kmh24k\" to finish creating: GET https://rp.j7942272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fcc9c987-dc72-47a5-acd2-7492bcbbcd08\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fcc9c987-dc72-47a5-acd2-7492bcbbcd08\",\n \"name\": \"fcc9c987-dc72-47a5-acd2-7492bcbbcd08\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:54:48.473384961Z\",\n \"endTime\": \"2026-03-16T14:56:32.581499648Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-202/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-202/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster ea-cluster: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-kmh24k\\\" to finish creating: GET https://rp.j7942272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fcc9c987-dc72-47a5-acd2-7492bcbbcd08\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fcc9c987-dc72-47a5-acd2-7492bcbbcd08\\\",\\n \\\"name\\\": \\\"fcc9c987-dc72-47a5-acd2-7492bcbbcd08\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:54:48.473384961Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:56:32.581499648Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-202/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-202/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000f407c0\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-kmh24k\\\" to finish creating: GET https://rp.j7942272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fcc9c987-dc72-47a5-acd2-7492bcbbcd08\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fcc9c987-dc72-47a5-acd2-7492bcbbcd08\\\",\\n \\\"name\\\": \\\"fcc9c987-dc72-47a5-acd2-7492bcbbcd08\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:54:48.473384961Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:56:32.581499648Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-202/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-202/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n----------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4211/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033516828721942528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4211,"pr_state":"open","pr_sha":"39e78504a6bf08c33aab10d3c70023229279c73e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:12:47Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"a80619d0b2db2c331a72a2b17ba403ca56ed5663ae07f5f87e1f0d9e92355577","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4211/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033516828721942528","occurred_at":"2026-03-16T12:12:47Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033516577046925312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4219,"pr_state":"closed","pr_sha":"c8926b8ff8e632c2a446e743641c7f2b0c5b7f57","final_merged_sha":"8550ece78eb685f5f0357a6937bf6b6e9f457543","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:11:47Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b69bc5ca393ad3ac175d88786f08b0357a636901ff77b4bbd65ebc3511638bfc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033516577046925312","occurred_at":"2026-03-16T12:11:47Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033516174842531840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4224,"pr_state":"closed","pr_sha":"89dbea9535e9da4e251791f532920158951c59f0","final_merged_sha":"0d2c310eee81cd65f33fd842930856c19dc160e3","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:10:11Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"cebff762f45b9e0dd2125dc6132dfabde70196cb623692d11ff964bfb9f7e0bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033516174842531840","occurred_at":"2026-03-16T12:10:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033515973075537920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4226,"pr_state":"closed","pr_sha":"3f588748fd29207330fe25d741a19c20e8d6fbb8","final_merged_sha":"138964dfc1d79a030eb9c07890259cb904e592a1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:09:23Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"61143663967d8f33d8b53728d9a5116bee3b6aeb3ae310e1b2b4eec779d9c38b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033515973075537920","occurred_at":"2026-03-16T12:09:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4234/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033515368122683392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4234,"pr_state":"closed","pr_sha":"9d6e88a80b9b97c82a5eaab0cba5b1c7d9148a13","final_merged_sha":"ac3354ace12fb48d60cedc2365d42cdcd0e45759","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:06:59Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"25fa99e9bb96893d2ffd8530a0747df559c0df4eff96edfda543f22e4d91a549","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4234/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033515368122683392","occurred_at":"2026-03-16T12:06:59Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4235/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033515267459387392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4235,"pr_state":"closed","pr_sha":"fea5b77c9fb0f5ab605e84f05f5890edc8162210","final_merged_sha":"fea5b77c9fb0f5ab605e84f05f5890edc8162210","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T12:06:35Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"712223e2ecd142ba5a159a89bddad930cda1f03a089925bb697953178de6073c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4235/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033515267459387392","occurred_at":"2026-03-16T12:06:35Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4241/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033515167882416128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4241,"pr_state":"closed","pr_sha":"f0eb454d49936e73b94a7710cef63a611155b521","final_merged_sha":"f0eb454d49936e73b94a7710cef63a611155b521","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T12:06:11Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b829d2375f7c172022eeb0a02ec90f6182746d3ad9f9b30ad13467da2b99229d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4241/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033515167882416128","occurred_at":"2026-03-16T12:06:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4255/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033514965448527872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4255,"pr_state":"open","pr_sha":"7ec0c2cf1ecf3837013224577c87c1158a02c4ea","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:05:23Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"d91e5d50a0f42b28b87e419e2681bcb86cad02f6ef384bd8dbddc895e966f60b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4255/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033514965448527872","occurred_at":"2026-03-16T12:05:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4257/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033514865758310400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4257,"pr_state":"closed","pr_sha":"77f6ef2ec4d628489e9b1b062e5279825a2a3cfd","final_merged_sha":"f9b0dd45c8c2499c52d47ec79f8297cdae4dfe1f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:04:59Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"6dcd53e5c67ea54756219627ee83aee6eedde931d6e4c1b87063ea1d1aa0687a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4257/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033514865758310400","occurred_at":"2026-03-16T12:04:59Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4272/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033514412848975872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4272,"pr_state":"open","pr_sha":"66a51589dab1c535676aba03930d89302e1c5b99","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:03:11Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b4be0c9bebcdc7e8bb7d7361ba0cfef3ae78143e4da5943502a902cb0bacb243","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4272/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033514412848975872","occurred_at":"2026-03-16T12:03:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4281/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033514102977990656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4281,"pr_state":"open","pr_sha":"f55caa5641a1457b14fd2f647226597c888b9484","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:01:35Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"be2dfdaa55ef5628fea74378423f72666f2fde536c6f9566adfc077eb44ba736","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4281/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033514102977990656","occurred_at":"2026-03-16T12:01:35Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4286/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033513842956308480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4286,"pr_state":"open","pr_sha":"39e398e22ce988f820254e230c5736d7cad47ded","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:00:47Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"a920ab9b9a7c76144f46bade0bfe26766566f1a55accd3b9cca6f98b7942475f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4286/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033513842956308480","occurred_at":"2026-03-16T12:00:47Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4291/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033513803492102144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4291,"pr_state":"closed","pr_sha":"932223aff8d18d5a75f34cc234dc10437a71ce6f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T12:00:23Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"8d19629b28a9ec6407a2f746e4b15c14a1cd5c04303ad57dec0e780836e233f1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4291/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033513803492102144","occurred_at":"2026-03-16T12:00:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033513506661208064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4305,"pr_state":"closed","pr_sha":"c0fed252385dfe86f5fac6579c94b69abfa267dd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:59:35Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e4318a68f9c3f6d14bc7719650240c12bdb455ce18404c3091a7b719fb63a84f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033513506661208064","occurred_at":"2026-03-16T11:59:35Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4342/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033512500074385408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4342,"pr_state":"closed","pr_sha":"4c2e431298b67e1a37ee2e8558c59b3da02d99c5","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:55:35Z"},"failed_at":["e2e"],"failed_test_count":16,"occurrences":[{"row_id":"02c5df243ed8dc3f458757d8e8f8b814c1c7dfc78bf88fad9a0714793bee9333","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4342/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033512500074385408","occurred_at":"2026-03-16T11:55:35Z","failed_at":"e2e","signature_id":"e4526e078a804fb5aeb4705d676d57e753b8f764f2e83d1b55b18d2c4af7db2d","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e80240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-nqc97l\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00110e480\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-nqc97l\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00110e460\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-nqc97l\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00110e440\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-nqc97l\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09d0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e80240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-nqc97l\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00110e480\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-nqc97l\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00110e460\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-nqc97l\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00110e440\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-nqc97l\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09d0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"21695b740bed030dcff1fe390fdf400ce97fa3301e0d324ed9e216fec723e439","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4342/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033512500074385408","occurred_at":"2026-03-16T11:55:35Z","failed_at":"e2e","signature_id":"3cf433a1e5b7a603ae582d119415220026c0163871a0a8c6d2d034772d0b52c0","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0012200d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-m4snht\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00075fc80\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-m4snht\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00075fc60\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-m4snht\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00075fc40\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-m4snht\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09d0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0012200d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-m4snht\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00075fc80\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-m4snht\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00075fc60\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-m4snht\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00075fc40\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-m4snht\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09d0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"2331bbd87be4220e014c79a56bfb36bbfce3cbc1d872f22b998ec95f65bb55a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4342/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033512500074385408","occurred_at":"2026-03-16T11:55:35Z","failed_at":"e2e","signature_id":"3eccd22334cd76663637654b8f7743f30a54ed96ec26b266f1a745430fdbb929","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00161e1f8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-jqdc5f\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000e82fa0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-jqdc5f\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e82f80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-jqdc5f\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e82f60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-jqdc5f\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09d0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00161e1f8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-jqdc5f\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000e82fa0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-jqdc5f\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e82f80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-jqdc5f\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e82f60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-jqdc5f\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09d0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"2814effb5cd7ee349f76053a13db08b3ebf15150d487b41a668382f9b2e14586","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4342/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033512500074385408","occurred_at":"2026-03-16T11:55:35Z","failed_at":"e2e","signature_id":"0da4f85201061686139d1cf5d42158a23bdcc6ad483b576f9125a66475e051a2","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00021a4c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-ksdsq5\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007a57e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-ksdsq5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007a57c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-ksdsq5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007a57a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-ksdsq5\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09d0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00021a4c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-ksdsq5\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007a57e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-ksdsq5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007a57c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-ksdsq5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007a57a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-ksdsq5\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09d0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"5cb59a7fdd223402d48973dea31f95181d105c6cb6695fc02014b497a9906474","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4342/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033512500074385408","occurred_at":"2026-03-16T11:55:35Z","failed_at":"e2e","signature_id":"1b9f7964d3e4839ed38d89ccfd52f2babb469681c46a3b4ded0d361756e9db9b","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f481c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-mzs9x9\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a776e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-mzs9x9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a776c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-mzs9x9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a776a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-mzs9x9\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09d0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f481c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-mzs9x9\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a776e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-mzs9x9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a776c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-mzs9x9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a776a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-mzs9x9\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09d0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"7610ee7335eeca098602fc01648d89ee3dd4cb57b5620e760ca34e187cd38116","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4342/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033512500074385408","occurred_at":"2026-03-16T11:55:35Z","failed_at":"e2e","signature_id":"60dd05606305dcbd2345d00aaa6c7547e90026ea732cc722b03de324e44811fb","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.003s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00093f6a0\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.003s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00093f6a0\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}},{"row_id":"810fd658378e50c0e8bd3d4a2ff5f739a85506423d646cfc39b5f1fb9d2f3af3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4342/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033512500074385408","occurred_at":"2026-03-16T11:55:35Z","failed_at":"e2e","signature_id":"d2e74f137be46e3554841a59e2db3e6652dac053f0a4cb5f488476d0922b3b49","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001774210\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-tcncf7\" in resourcegroup=\"rg-zstream-upgrade-4-21-zs86xz\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000d413e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-tcncf7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-zs86xz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d413c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-tcncf7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-zs86xz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d413a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-tcncf7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-zs86xz\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09d0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001774210\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-tcncf7\" in resourcegroup=\"rg-zstream-upgrade-4-21-zs86xz\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000d413e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-tcncf7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-zs86xz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d413c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-tcncf7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-zs86xz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d413a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-tcncf7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-zs86xz\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09d0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"818407041fe7395841535a6fdf8d64dd6a19f7bc04e32a9c8bcc0c5f8c9ed6cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4342/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033512500074385408","occurred_at":"2026-03-16T11:55:35Z","failed_at":"e2e","signature_id":"5b172da6c132571897e7146ecbb7f20d612187c2b310e894121782ad58e1bba9","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:96]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a0fee0\u003e: \n failed to create HCP cluster basic-hcp-cluster: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-tm8z9r\" to finish creating: GET https://rp.j4385408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/276ecb68-8f36-4788-a27a-b959fd469eb8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/276ecb68-8f36-4788-a27a-b959fd469eb8\",\n \"name\": \"276ecb68-8f36-4788-a27a-b959fd469eb8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:30:45.201431526Z\",\n \"endTime\": \"2026-03-16T14:31:35.067401014Z\",\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"Cluster provisioning failed due to multiple errors\",\n \"details\": [\n {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-tm8z9r/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n },\n {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-131/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-131/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster basic-hcp-cluster: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-tm8z9r\\\" to finish creating: GET https://rp.j4385408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/276ecb68-8f36-4788-a27a-b959fd469eb8\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/276ecb68-8f36-4788-a27a-b959fd469eb8\\\",\\n \\\"name\\\": \\\"276ecb68-8f36-4788-a27a-b959fd469eb8\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:30:45.201431526Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:31:35.067401014Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-tm8z9r/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n },\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-131/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-131/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a0fec0\u003e{\n msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-tm8z9r\\\" to finish creating: GET https://rp.j4385408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/276ecb68-8f36-4788-a27a-b959fd469eb8\\n-----------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:96]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a0fee0\u003e: \n failed to create HCP cluster basic-hcp-cluster: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-tm8z9r\" to finish creating: GET https://rp.j4385408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/276ecb68-8f36-4788-a27a-b959fd469eb8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/276ecb68-8f36-4788-a27a-b959fd469eb8\",\n \"name\": \"276ecb68-8f36-4788-a27a-b959fd469eb8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:30:45.201431526Z\",\n \"endTime\": \"2026-03-16T14:31:35.067401014Z\",\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"Cluster provisioning failed due to multiple errors\",\n \"details\": [\n {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-tm8z9r/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n },\n {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-131/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-131/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster basic-hcp-cluster: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-tm8z9r\\\" to finish creating: GET https://rp.j4385408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/276ecb68-8f36-4788-a27a-b959fd469eb8\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/276ecb68-8f36-4788-a27a-b959fd469eb8\\\",\\n \\\"name\\\": \\\"276ecb68-8f36-4788-a27a-b959fd469eb8\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:30:45.201431526Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:31:35.067401014Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-tm8z9r/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n },\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-131/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-131/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a0fec0\u003e{\n msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-tm8z9r\\\" to finish creating: GET https://rp.j4385408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/276ecb68-8f36-4788-a27a-b959fd469eb8\\n-----------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0aa9ed5d94e7167a12aa30282eff19539fc3fc473772cb96b223d420df66775f","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"86c700d3a5a44d12b3f6da455fb88b1cbd4dd13ff6ed0efe14755d917d195c25","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4342/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033512500074385408","occurred_at":"2026-03-16T11:55:35Z","failed_at":"e2e","signature_id":"679281e76e86a73af92a1e4cec9341832af19a59e60bfa54825f5ae248cfe340","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000fd6228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-g9db66\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f87640\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-g9db66\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f87600\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-g9db66\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f875e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-g9db66\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09d0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000fd6228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-g9db66\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f87640\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-g9db66\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f87600\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-g9db66\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f875e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-g9db66\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09d0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"8c3525f698e0738831178a7d42198bf7ff63787a5cd1382bb1a435ebe76995ce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4342/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033512500074385408","occurred_at":"2026-03-16T11:55:35Z","failed_at":"e2e","signature_id":"60a6e195ed9920eafa3ac3b310d9203960e91355c1fa831a66070be6dcdbd37d","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001590108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-9q9dbq\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0006e8e20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-9q9dbq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006e8e00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-9q9dbq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006e8de0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-9q9dbq\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09d0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001590108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-9q9dbq\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0006e8e20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-9q9dbq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006e8e00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-9q9dbq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006e8de0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-9q9dbq\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09d0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"8c775aaf06b1b74a52fa3f33cb779ff2aec73b6ce7937548a4baa77ebd302703","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4342/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033512500074385408","occurred_at":"2026-03-16T11:55:35Z","failed_at":"e2e","signature_id":"59de7d9282084b48fb641f929c46f7748fa7ee11d905b918929c280129ff3860","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0007d0228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-mlbqd7\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000808660\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-mlbqd7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000808640\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-mlbqd7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000808620\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-mlbqd7\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09d0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0007d0228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-mlbqd7\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000808660\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-mlbqd7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000808640\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-mlbqd7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000808620\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-mlbqd7\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09d0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"97a17ace2abecbfc52739a6dd6e793cc57437d720beaed9541624a9ff9f5786d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4342/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033512500074385408","occurred_at":"2026-03-16T11:55:35Z","failed_at":"e2e","signature_id":"c0f483eada90887d0e93a0b0b00554ead3736d04dcee04c7d5a721b093e03516","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001780120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-rxbrqv\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001423540\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-rxbrqv\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001423520\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-rxbrqv\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001423500\u003e{\n msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-rxbrqv\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09d0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001780120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-rxbrqv\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001423540\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-rxbrqv\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001423520\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-rxbrqv\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001423500\u003e{\n msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-rxbrqv\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09d0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"cb9614547266f981698c1dca920083b5c061b2ca9ed503e0432ff3de63bdb3da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4342/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033512500074385408","occurred_at":"2026-03-16T11:55:35Z","failed_at":"e2e","signature_id":"ddb16a5cfd74c217f667fba56590a2c0af3ed3dee8429aa2b370f5ada0fa4d07","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f4c2e8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-rflvdb\" in resourcegroup=\"admin-credential-lifecycle-test-rx8cq2\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000e5aec0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-rflvdb\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-rx8cq2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e5aea0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-rflvdb\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-rx8cq2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e5ae80\u003e{\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-rflvdb\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-rx8cq2\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09d0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f4c2e8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-rflvdb\" in resourcegroup=\"admin-credential-lifecycle-test-rx8cq2\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000e5aec0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-rflvdb\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-rx8cq2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e5aea0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-rflvdb\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-rx8cq2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e5ae80\u003e{\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-rflvdb\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-rx8cq2\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09d0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"d4825ddc587b6d9e7b3727288acd80394419e52099661f9ada74fb831f8089cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4342/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033512500074385408","occurred_at":"2026-03-16T11:55:35Z","failed_at":"e2e","signature_id":"010b75c2fdc61c18ad589076eb7a7ecf0d5e893783dfab2ef08527b27e322901","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001264100\u003e: \n failed to create HCP cluster cluster-zstream-4-20-2h67vc: failed waiting for cluster=\"cluster-zstream-4-20-2h67vc\" in resourcegroup=\"rg-zstream-upgrade-4-20-jmj74s\" to finish creating: GET https://rp.j4385408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5591d220-8f1b-4485-99fc-95c50a90e42c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5591d220-8f1b-4485-99fc-95c50a90e42c\",\n \"name\": \"5591d220-8f1b-4485-99fc-95c50a90e42c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T15:00:20.941704343Z\",\n \"endTime\": \"2026-03-16T15:01:07.240406639Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-65/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-65/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-20-2h67vc: failed waiting for cluster=\\\"cluster-zstream-4-20-2h67vc\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-jmj74s\\\" to finish creating: GET https://rp.j4385408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5591d220-8f1b-4485-99fc-95c50a90e42c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5591d220-8f1b-4485-99fc-95c50a90e42c\\\",\\n \\\"name\\\": \\\"5591d220-8f1b-4485-99fc-95c50a90e42c\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T15:00:20.941704343Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T15:01:07.240406639Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-65/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-65/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0012640e0\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-2h67vc\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-jmj74s\\\" to finish creating: GET https://rp.j4385408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5591d220-8f1b-4485-99fc-95c50a90e42c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5591d220-8f1b-4485-99fc-95c50a90e42c\\\",\\n \\\"name\\\": \\\"5591d220-8f1b-4485-99fc-95c50a90e42c\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T15:00:20.941704343Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T15:01:07.240406639Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-65/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-65/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001264100\u003e: \n failed to create HCP cluster cluster-zstream-4-20-2h67vc: failed waiting for cluster=\"cluster-zstream-4-20-2h67vc\" in resourcegroup=\"rg-zstream-upgrade-4-20-jmj74s\" to finish creating: GET https://rp.j4385408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5591d220-8f1b-4485-99fc-95c50a90e42c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5591d220-8f1b-4485-99fc-95c50a90e42c\",\n \"name\": \"5591d220-8f1b-4485-99fc-95c50a90e42c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T15:00:20.941704343Z\",\n \"endTime\": \"2026-03-16T15:01:07.240406639Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-65/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-65/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-20-2h67vc: failed waiting for cluster=\\\"cluster-zstream-4-20-2h67vc\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-jmj74s\\\" to finish creating: GET https://rp.j4385408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5591d220-8f1b-4485-99fc-95c50a90e42c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5591d220-8f1b-4485-99fc-95c50a90e42c\\\",\\n \\\"name\\\": \\\"5591d220-8f1b-4485-99fc-95c50a90e42c\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T15:00:20.941704343Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T15:01:07.240406639Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-65/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-65/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0012640e0\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-2h67vc\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-jmj74s\\\" to finish creating: GET https://rp.j4385408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5591d220-8f1b-4485-99fc-95c50a90e42c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5591d220-8f1b-4485-99fc-95c50a90e42c\\\",\\n \\\"name\\\": \\\"5591d220-8f1b-4485-99fc-95c50a90e42c\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T15:00:20.941704343Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T15:01:07.240406639Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-65/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-65/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d54c41e79c895ad3b6754e6b1e1173c29edf4ae7ba98c1b34b785d342d89fb96","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4342/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033512500074385408","occurred_at":"2026-03-16T11:55:35Z","failed_at":"e2e","signature_id":"e400a3182fa13db53fefa0ba60d045e31ffb6c62e5b63be842013e2285873a42","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ffe2e8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-wk26tz\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00069fc80\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-wk26tz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00069fc60\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-wk26tz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00069fc20\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-wk26tz\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09d0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ffe2e8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-wk26tz\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00069fc80\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-wk26tz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00069fc60\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-wk26tz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00069fc20\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-wk26tz\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09d0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"d905da5f2138a5c6f78c3796f4a0ea96a70c39ea0fd24400c6841f54688aa3c7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4342/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033512500074385408","occurred_at":"2026-03-16T11:55:35Z","failed_at":"e2e","signature_id":"97ab86cc117f209cdd98b81a3b596d9deab064d5ed4650692728e3237cf2c088","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:86]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0004930a0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-hf9qjl\" to finish creating: GET https://rp.j4385408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21c9583a-7160-4b35-a5ee-f381fc0ba33b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21c9583a-7160-4b35-a5ee-f381fc0ba33b\",\n \"name\": \"21c9583a-7160-4b35-a5ee-f381fc0ba33b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:55:06.527348633Z\",\n \"endTime\": \"2026-03-16T14:55:50.172982114Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-233/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-233/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-hf9qjl\\\" to finish creating: GET https://rp.j4385408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21c9583a-7160-4b35-a5ee-f381fc0ba33b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21c9583a-7160-4b35-a5ee-f381fc0ba33b\\\",\\n \\\"name\\\": \\\"21c9583a-7160-4b35-a5ee-f381fc0ba33b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:55:06.527348633Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:55:50.172982114Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-233/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-233/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000493060\u003e{\n msg: \"failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-hf9qjl\\\" to finish creating: GET https://rp.j4385408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21c9583a-7160-4b35-a5ee-f381fc0ba33b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21c9583a-7160-4b35-a5ee-f381fc0ba33b\\\",\\n \\\"name\\\": \\\"21c9583a-7160-4b35-a5ee-f381fc0ba33b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:55:06.527348633Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:55:50.172982114Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-233/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-233/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-----------------------------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:86]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0004930a0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-hf9qjl\" to finish creating: GET https://rp.j4385408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21c9583a-7160-4b35-a5ee-f381fc0ba33b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21c9583a-7160-4b35-a5ee-f381fc0ba33b\",\n \"name\": \"21c9583a-7160-4b35-a5ee-f381fc0ba33b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:55:06.527348633Z\",\n \"endTime\": \"2026-03-16T14:55:50.172982114Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-233/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-233/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-hf9qjl\\\" to finish creating: GET https://rp.j4385408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21c9583a-7160-4b35-a5ee-f381fc0ba33b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21c9583a-7160-4b35-a5ee-f381fc0ba33b\\\",\\n \\\"name\\\": \\\"21c9583a-7160-4b35-a5ee-f381fc0ba33b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:55:06.527348633Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:55:50.172982114Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-233/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-233/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000493060\u003e{\n msg: \"failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-hf9qjl\\\" to finish creating: GET https://rp.j4385408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21c9583a-7160-4b35-a5ee-f381fc0ba33b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21c9583a-7160-4b35-a5ee-f381fc0ba33b\\\",\\n \\\"name\\\": \\\"21c9583a-7160-4b35-a5ee-f381fc0ba33b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:55:06.527348633Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:55:50.172982114Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-233/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-233/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-----------------------------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":16,"failure_pattern_ids":["0aa9ed5d94e7167a12aa30282eff19539fc3fc473772cb96b223d420df66775f","503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7"],"matched_occurrences":16,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4343/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033512298630352896","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4343,"pr_state":"closed","pr_sha":"7f9a7eb1138d7d70c0f4c2e8c0d91bac822a7048","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:54:47Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"6bd98c952719e8305dab9e883b9dec44d37eb1130e60a51ab02c831422f126b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4343/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033512298630352896","occurred_at":"2026-03-16T11:54:47Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4344/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033512097417007104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4344,"pr_state":"open","pr_sha":"73b7fffa9a443b3dcc3ef15c6bf38965981cb423","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:53:59Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3184d684fe5f17f0f434f364f92050f7dfcc75751333f1a50aa70d0ddfbd5c2c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4344/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033512097417007104","occurred_at":"2026-03-16T11:53:59Z","failed_at":"provision","signature_id":"3bc00be601ea03baa3af8c6e65a7203a7cf06f70ba6ffe065eb0eb25ba5e3d96","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.HypershiftOperator/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-03-16T14:23:17.605Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.HypershiftOperator resourceGroup=global step=mirror-image description=\"Step mirror-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e hypershift.image.registry \u003cnil\u003e}:{\u003cnil\u003e hypershift.image.repository \u003cnil\u003e}@{\u003cnil\u003e hypershift.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-16T14:23:41.134Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.HypershiftOperator resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org\\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nFailed to log in to ACR arohcpsvcdev: ERROR: The command failed with an unexpected error. Here is the traceback:\\nERROR: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /oauth2/exchange (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 204, in _new_conn\\n sock = connection.create_connection(\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/util/connection.py\\\", line 60, in create_connection\\n for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/python3.12/socket.py\\\", line 963, in getaddrinfo\\n for res in _socket.getaddrinfo(host, port, family, type, proto, flags):\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\nsocket.gaierror: [Errno -2] Name or service not known\\n\\nThe above exception was the direct cause of the following exception:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 787, in urlopen\\n response = self._make_request(\\n ^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 488, in _make_request\\n raise new_e\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 464, in _make_request\\n self._validate_conn(conn)\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 1093, in _validate_conn\\n conn.connect()\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 759, in connect\\n self.sock = sock = self._new_conn()\\n ^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 211, in _new_conn\\n raise NameResolutionError(self.host, self, e) from e\\nurllib3.exceptions.NameResolutionError: HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\n\\nThe above exception was the direct cause of the following exception:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/adapters.py\\\", line 667, in send\\n resp = conn.urlopen(\\n ^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 841, in urlopen\\n retries = retries.increment(\\n ^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/util/retry.py\\\", line 535, in increment\\n raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\nurllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /oauth2/exchange (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\n\\nDuring handling of the above exception, another exception occurred:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/knack/cli.py\\\", line 233, in invoke\\n cmd_result = self.invocation.execute(args)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 682, in execute\\n raise ex\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 812, in _run_jobs_serially\\n results.append(self._run_job(expanded_arg, cmd_copy))\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 781, in _run_job\\n result = cmd_copy(params)\\n ^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 336, in __call__\\n return self.handler(*args, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/command_operation.py\\\", line 120, in handler\\n return op(**command_args)\\n ^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/custom.py\\\", line 294, in acr_login\\n login_server, _, password = get_login_credentials(\\n ^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 505, in get_login_credentials\\n return _get_credentials(cmd,\\n ^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 449, in _get_credentials\\n return login_server, EMPTY_GUID, _get_aad_token(cli_ctx,\\n ^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 285, in _get_aad_token\\n return _get_aad_token_after_challenge(cli_ctx,\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 159, in _get_aad_token_after_challenge\\n response = requests.post(url=authhost, data=urlencode(content), headers=headers,\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/api.py\\\", line 115, in post\\n return request(\\\"post\\\", url, data=data, json=json, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/api.py\\\", line 59, in request\\n return session.request(method=method, url=url, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/sessions.py\\\", line 589, in request\\n resp = self.send(prep, **send_kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/sessions.py\\\", line 703, in send\\n r = adapter.send(request, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/adapters.py\\\", line 700, in send\\n raise ConnectionError(e, request=request)\\nrequests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /oauth2/exchange (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\nTo check existing issues, please visit: https://github.com/Azure/azure-cli/issues\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9a2a4cf3344f5eab04c519ebc438e0419dbf08c2ef1692b2ed6fd7228e53ded3","failure_pattern":"requests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /oauth2/exchange (Caused by NameResolutionError(\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve","search_query":"requests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /oauth2/exchange (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9a2a4cf3344f5eab04c519ebc438e0419dbf08c2ef1692b2ed6fd7228e53ded3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4346/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511896065249280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4346,"pr_state":"closed","pr_sha":"d7ca9a9e28a45a22679d375be63783f08cf04f47","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:53:11Z"},"failed_at":["e2e"],"failed_test_count":19,"occurrences":[{"row_id":"1f5d86f827fbc783961c8420020b872bed9e39ba06c51764ef85e5fcf5e24354","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4346/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511896065249280","occurred_at":"2026-03-16T11:53:11Z","failed_at":"e2e","signature_id":"93acc17751164af6fee49b2e78e62faa609bdb9bdf79ff2c9561c01b3bf3febf","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:73]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0014d62c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-update-nodes-hcp-cluster-pp66nd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-pp66nd/operationStatuses/08584279348069236799\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-137/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-2mq2n4/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-np-update-nodes-hcp-cluster-pp66nd\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-pp66nd/operationStatuses/08584279348069236799\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-137/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-2mq2n4/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:73]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0014d62c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-update-nodes-hcp-cluster-pp66nd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-pp66nd/operationStatuses/08584279348069236799\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-137/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-2mq2n4/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-np-update-nodes-hcp-cluster-pp66nd\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-pp66nd/operationStatuses/08584279348069236799\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-137/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-2mq2n4/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"22fd8347ac2fb93f0cefd0e77641d02ee1f97118f34fb24cb39c63c39025a53a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4346/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511896065249280","occurred_at":"2026-03-16T11:53:11Z","failed_at":"e2e","signature_id":"6fd9012db2202eba33f5bf498786fff8b3e66481fc94264182f9bf447244aae6","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0016b81e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0009d13a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-22nlpz\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-69cfx2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009d1360\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-22nlpz\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-69cfx2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009d1340\u003e{\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-22nlpz\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-69cfx2\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0016b81e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0009d13a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-22nlpz\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-69cfx2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009d1360\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-22nlpz\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-69cfx2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009d1340\u003e{\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-22nlpz\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-69cfx2\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"41f7a527d7d4ae814c1357497906e267f74e0d04e6335bcf9b29f15fee4996cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4346/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511896065249280","occurred_at":"2026-03-16T11:53:11Z","failed_at":"e2e","signature_id":"54d59a99a53887f534b901af64cc495b9e41b57e094ae598562f5272fbe33749","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000b7c750\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-19-sxbwlr\" in resourcegroup=\"rg-zstream-upgrade-4-19-gvfq6z\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00079ca00\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-19-sxbwlr\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-gvfq6z\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00079c9e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-19-sxbwlr\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-gvfq6z\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00079c9c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-sxbwlr\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-gvfq6z\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000b7c750\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-19-sxbwlr\" in resourcegroup=\"rg-zstream-upgrade-4-19-gvfq6z\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00079ca00\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-19-sxbwlr\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-gvfq6z\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00079c9e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-19-sxbwlr\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-gvfq6z\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00079c9c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-sxbwlr\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-gvfq6z\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"4600f7f48133bc88de81e6e7a2d54c691276ebc6562fe85c23b7cb8e7b12d0db","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4346/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511896065249280","occurred_at":"2026-03-16T11:53:11Z","failed_at":"e2e","signature_id":"9a381a3cf8409171fd49a07fa71588e215f88252f9a746f568e8a19fc55518c3","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0015b60f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-dvpw9w\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000e06ce0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-dvpw9w\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e06cc0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-dvpw9w\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e06ca0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-dvpw9w\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0015b60f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-dvpw9w\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000e06ce0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-dvpw9w\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e06cc0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-dvpw9w\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e06ca0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-dvpw9w\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"4c09440aec874288e94ac15721347f5b528aef9985d607ace11bc7c7b857e7fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4346/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511896065249280","occurred_at":"2026-03-16T11:53:11Z","failed_at":"e2e","signature_id":"9951b0906a5654c34400f253731ce8a0a89ed52d89cca6e638597d802a085da0","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e10240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-fm7r92\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0016e0000\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-fm7r92\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b5bf60\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-fm7r92\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b5bf40\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-fm7r92\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e10240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-fm7r92\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0016e0000\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-fm7r92\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b5bf60\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-fm7r92\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b5bf40\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-fm7r92\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"5a5aa129c3ac7b4cfdfdef1464c25e53c4dfd5040f4fd81d32a916bd0cd0749f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4346/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511896065249280","occurred_at":"2026-03-16T11:53:11Z","failed_at":"e2e","signature_id":"17df2e30b3e98af3baf030678b71d7a318bd7f936fcffe4e0fd2af93c6c2368f","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00108c2a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-47z72f\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a95100\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-47z72f\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a950e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-47z72f\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a950c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-47z72f\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00108c2a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-47z72f\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a95100\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-47z72f\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a950e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-47z72f\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a950c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-47z72f\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"6d658317f2d1c5adb475d210ca6c54621d603d67adbbf1bad7abef82606e3f75","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4346/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511896065249280","occurred_at":"2026-03-16T11:53:11Z","failed_at":"e2e","signature_id":"27227ad1924254d869074fe6ff921dafa02639ea5c3d0d7293d39d87bc405e0a","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a282a0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-pbt42n: failed waiting for cluster=\"cluster-zstream-4-21-pbt42n\" in resourcegroup=\"rg-zstream-upgrade-4-21-9lv5nh\" to finish creating: GET https://rp.j5249280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc08e50a-b558-47b3-9bce-e9c0b75671b8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc08e50a-b558-47b3-9bce-e9c0b75671b8\",\n \"name\": \"dc08e50a-b558-47b3-9bce-e9c0b75671b8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:38:41.910090205Z\",\n \"endTime\": \"2026-03-16T14:39:43.777378228Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-203/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-203/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-21-pbt42n: failed waiting for cluster=\\\"cluster-zstream-4-21-pbt42n\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-9lv5nh\\\" to finish creating: GET https://rp.j5249280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc08e50a-b558-47b3-9bce-e9c0b75671b8\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc08e50a-b558-47b3-9bce-e9c0b75671b8\\\",\\n \\\"name\\\": \\\"dc08e50a-b558-47b3-9bce-e9c0b75671b8\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:38:41.910090205Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:39:43.777378228Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-203/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-203/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a28260\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-pbt42n\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-9lv5nh\\\" to finish creating: GET https://rp.j5249280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc08e50a-b558-47b3-9bce-e9c0b75671b8\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc08e50a-b558-47b3-9bce-e9c0b75671b8\\\",\\n \\\"name\\\": \\\"dc08e50a-b558-47b3-9bce-e9c0b75671b8\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:38:41.910090205Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:39:43.777378228Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-203/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-203/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: Subsc...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a282a0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-pbt42n: failed waiting for cluster=\"cluster-zstream-4-21-pbt42n\" in resourcegroup=\"rg-zstream-upgrade-4-21-9lv5nh\" to finish creating: GET https://rp.j5249280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc08e50a-b558-47b3-9bce-e9c0b75671b8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc08e50a-b558-47b3-9bce-e9c0b75671b8\",\n \"name\": \"dc08e50a-b558-47b3-9bce-e9c0b75671b8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:38:41.910090205Z\",\n \"endTime\": \"2026-03-16T14:39:43.777378228Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-203/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-203/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-21-pbt42n: failed waiting for cluster=\\\"cluster-zstream-4-21-pbt42n\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-9lv5nh\\\" to finish creating: GET https://rp.j5249280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc08e50a-b558-47b3-9bce-e9c0b75671b8\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc08e50a-b558-47b3-9bce-e9c0b75671b8\\\",\\n \\\"name\\\": \\\"dc08e50a-b558-47b3-9bce-e9c0b75671b8\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:38:41.910090205Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:39:43.777378228Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-203/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-203/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a28260\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-pbt42n\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-9lv5nh\\\" to finish creating: GET https://rp.j5249280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc08e50a-b558-47b3-9bce-e9c0b75671b8\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc08e50a-b558-47b3-9bce-e9c0b75671b8\\\",\\n \\\"name\\\": \\\"dc08e50a-b558-47b3-9bce-e9c0b75671b8\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:38:41.910090205Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:39:43.777378228Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-203/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-203/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: Subsc...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"716769665e7cc6b2c343ed175d426e49de4e16f9f5748e63e013ca6d175bfa97","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4346/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511896065249280","occurred_at":"2026-03-16T11:53:11Z","failed_at":"e2e","signature_id":"821d81c668706f1774bec65ce83b264ddafaefce6fb5903bb879563e3c56c9cb","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e24690\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-w4wcl2\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000c97ec0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-w4wcl2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000c97e80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-w4wcl2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000c97e60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-w4wcl2\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e24690\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-w4wcl2\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000c97ec0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-w4wcl2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000c97e80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-w4wcl2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000c97e60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-w4wcl2\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"84e63a3608ee2cff6adbc21700c1c62c106dd40df3554aeb21d6c80510fdb271","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4346/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511896065249280","occurred_at":"2026-03-16T11:53:11Z","failed_at":"e2e","signature_id":"200fab97419ee03852608770a7bf669f15b0a6eab1288206cb377ee16bb33e77","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:390]: Timed out after 600.002s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:389 with:\nAll ClusterOperators should report Available=True, but these are not available: [image-registry (Available=False)]\nExpected\n \u003c[]string | len:1, cap:1\u003e: [\n \"image-registry (Available=False)\",\n ]\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:390]: Timed out after 600.002s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:389 with:\nAll ClusterOperators should report Available=True, but these are not available: [image-registry (Available=False)]\nExpected\n \u003c[]string | len:1, cap:1\u003e: [\n \"image-registry (Available=False)\",\n ]\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d24e138f3b84b77b0ea6793407ac1f41343d0658f0c69312eac159e4f4f7eb45","failure_pattern":"All ClusterOperators should report Available=True, but these are not available: [image-registry (Available=False)]","search_query":"All ClusterOperators should report Available=True, but these are not available: [image-registry (Available=False)]"}},{"row_id":"8b61c4fad1d5a8376becf7aa0bd7f05f45a240c2542e29873d547dcadb2c7953","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4346/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511896065249280","occurred_at":"2026-03-16T11:53:11Z","failed_at":"e2e","signature_id":"6009452f7fcd7e7f23dc315d36761aa1625445c0557b8ab0c6c7acc53a896366","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0015860f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-jgrbzg\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f27ac0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-jgrbzg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f27aa0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-jgrbzg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f27a80\u003e{\n msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-jgrbzg\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0015860f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-jgrbzg\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f27ac0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-jgrbzg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f27aa0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-jgrbzg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f27a80\u003e{\n msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-jgrbzg\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"9b39989b56774ffb24ee710bdcbcd9d84e35a5f7e37ca1482bf93e5c208f59da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4346/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511896065249280","occurred_at":"2026-03-16T11:53:11Z","failed_at":"e2e","signature_id":"15f00e83bd39997708b3df671f71e08b0344406fe2373f00e30b3d5255ba04b1","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a741e0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-8kc9g7: failed waiting for cluster=\"cluster-zstream-4-20-8kc9g7\" in resourcegroup=\"rg-zstream-upgrade-4-20-dcjhgg\" to finish creating: GET https://rp.j5249280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/724b22a8-140c-4634-bdee-1da54bd44a5f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/724b22a8-140c-4634-bdee-1da54bd44a5f\",\n \"name\": \"724b22a8-140c-4634-bdee-1da54bd44a5f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:36:41.379585924Z\",\n \"endTime\": \"2026-03-16T14:37:40.220183063Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-137/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-137/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-20-8kc9g7: failed waiting for cluster=\\\"cluster-zstream-4-20-8kc9g7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-dcjhgg\\\" to finish creating: GET https://rp.j5249280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/724b22a8-140c-4634-bdee-1da54bd44a5f\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/724b22a8-140c-4634-bdee-1da54bd44a5f\\\",\\n \\\"name\\\": \\\"724b22a8-140c-4634-bdee-1da54bd44a5f\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:36:41.379585924Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:37:40.220183063Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-137/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-137/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a74180\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-8kc9g7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-dcjhgg\\\" to finish creating: GET https://rp.j5249280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/724b22a8-140c-4634-bdee-1da54bd44a5f\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/724b22a8-140c-4634-bdee-1da54bd44a5f\\\",\\n \\\"name\\\": \\\"724b22a8-140c-4634-bdee-1da54bd44a5f\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:36:41.379585924Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:37:40.220183063Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-137/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-137/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a741e0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-8kc9g7: failed waiting for cluster=\"cluster-zstream-4-20-8kc9g7\" in resourcegroup=\"rg-zstream-upgrade-4-20-dcjhgg\" to finish creating: GET https://rp.j5249280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/724b22a8-140c-4634-bdee-1da54bd44a5f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/724b22a8-140c-4634-bdee-1da54bd44a5f\",\n \"name\": \"724b22a8-140c-4634-bdee-1da54bd44a5f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:36:41.379585924Z\",\n \"endTime\": \"2026-03-16T14:37:40.220183063Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-137/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-137/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-20-8kc9g7: failed waiting for cluster=\\\"cluster-zstream-4-20-8kc9g7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-dcjhgg\\\" to finish creating: GET https://rp.j5249280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/724b22a8-140c-4634-bdee-1da54bd44a5f\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/724b22a8-140c-4634-bdee-1da54bd44a5f\\\",\\n \\\"name\\\": \\\"724b22a8-140c-4634-bdee-1da54bd44a5f\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:36:41.379585924Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:37:40.220183063Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-137/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-137/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a74180\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-8kc9g7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-dcjhgg\\\" to finish creating: GET https://rp.j5249280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/724b22a8-140c-4634-bdee-1da54bd44a5f\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/724b22a8-140c-4634-bdee-1da54bd44a5f\\\",\\n \\\"name\\\": \\\"724b22a8-140c-4634-bdee-1da54bd44a5f\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:36:41.379585924Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:37:40.220183063Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-137/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-137/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b4f660cd685282fa5d26dcf784dc6c9f7cfdb165d0a63bd9d6e94ba855a4f824","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4346/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511896065249280","occurred_at":"2026-03-16T11:53:11Z","failed_at":"e2e","signature_id":"9f40e01812d7555ffbdd06d17765fbeb998737eb131e0c808ddb93d9916bf35f","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0014821e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-7q2hlp\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007ebcc0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-7q2hlp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007ebca0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-7q2hlp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007ebc60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-7q2hlp\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0014821e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-7q2hlp\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007ebcc0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-7q2hlp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007ebca0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-7q2hlp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007ebc60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-7q2hlp\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"b74f4bde9955f91cb5674c9d166c7dc816d26363574dc4a06ba740a9dd728407","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4346/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511896065249280","occurred_at":"2026-03-16T11:53:11Z","failed_at":"e2e","signature_id":"51bf57cabc5202891b25b3315f8c4b582a559045d2654dd8098a21f44addb8d7","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00102e2e8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-r72ngj\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000d0d6a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-r72ngj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d0d680\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-r72ngj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d0d660\u003e{\n msg: \"failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-r72ngj\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00102e2e8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-r72ngj\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000d0d6a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-r72ngj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d0d680\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-r72ngj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d0d660\u003e{\n msg: \"failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-r72ngj\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"bd66a27b8fa0c5d06a23ac33bf359b16c5f080188d765047d20212bdf0fb8088","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4346/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511896065249280","occurred_at":"2026-03-16T11:53:11Z","failed_at":"e2e","signature_id":"b3ec640cf82fee20244270977a02fb9a80cc523d10edacaaa5fecd26cfd531f4","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001760228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-x85bgb\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0014a3000\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-x85bgb\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0014a2fe0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-x85bgb\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0014a2fc0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-x85bgb\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001760228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-x85bgb\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0014a3000\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-x85bgb\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0014a2fe0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-x85bgb\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0014a2fc0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-x85bgb\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"c27b7ce396483dfae4cc1ac6559a6e9cb20b0aba91ae503d18442fdfc33ec079","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4346/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511896065249280","occurred_at":"2026-03-16T11:53:11Z","failed_at":"e2e","signature_id":"507ad756e8d1e57ed7cddc8bf2aa37f02006d259fa7a00485f3357cd1fae3b54","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0009f64c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-5rf8lf\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000617060\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-5rf8lf\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000617020\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-5rf8lf\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000617000\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-5rf8lf\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0009f64c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-5rf8lf\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000617060\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-5rf8lf\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000617020\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-5rf8lf\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000617000\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-5rf8lf\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"c987618833e0871caa4124b42a8bab467fb7d7b9df72fbcc2ad22e80e9db5994","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4346/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511896065249280","occurred_at":"2026-03-16T11:53:11Z","failed_at":"e2e","signature_id":"6df8c29fd8186c0f19341e7e5052fe356f38640dd5e0f96fe509cd883b7ffc48","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.007s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000518580\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.007s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000518580\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}},{"row_id":"d2e2d05c3507d800a865cc7c772e5e377e0a7b3cef4af3cc3fdbce968eeea4e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4346/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511896065249280","occurred_at":"2026-03-16T11:53:11Z","failed_at":"e2e","signature_id":"38ac0605c238508f626536a49ffeca632c613fa9eb5c9e9e5052955a8b4bcd56","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00105c0f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-ml49jg\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0013aa120\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-ml49jg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0013aa100\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-ml49jg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0013aa0e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-ml49jg\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00105c0f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-ml49jg\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0013aa120\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-ml49jg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0013aa100\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-ml49jg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0013aa0e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-ml49jg\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"e9bde2edd1ebee2765159879de1ff603ba52c2c4a8240db6d180d2cc14f4bff7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4346/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511896065249280","occurred_at":"2026-03-16T11:53:11Z","failed_at":"e2e","signature_id":"0cd665b504891c2ff056ea647fdf1672138b5e20bb8b39b02d9fcbb32b4a5337","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010b4408\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listx2wbnq\" in resourcegroup=\"ea-list-9vgbb7\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0013ed380\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listx2wbnq\\\" in resourcegroup=\\\"ea-list-9vgbb7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0013ed360\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listx2wbnq\\\" in resourcegroup=\\\"ea-list-9vgbb7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0013ed340\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listx2wbnq\\\" in resourcegroup=\\\"ea-list-9vgbb7\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010b4408\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listx2wbnq\" in resourcegroup=\"ea-list-9vgbb7\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0013ed380\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listx2wbnq\\\" in resourcegroup=\\\"ea-list-9vgbb7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0013ed360\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listx2wbnq\\\" in resourcegroup=\\\"ea-list-9vgbb7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0013ed340\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listx2wbnq\\\" in resourcegroup=\\\"ea-list-9vgbb7\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"eb2cae8e953e7666f6558ba1154b0fcc5820b0c9758c62cb8b8459bf39b3764e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4346/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511896065249280","occurred_at":"2026-03-16T11:53:11Z","failed_at":"e2e","signature_id":"7477859a6d8c97249e48d6f645b4f33e54ca3f7dbe7874d3ea0fc5d10f92b122","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0009603d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-k2wbnb\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0009e54e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-k2wbnb\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009e54c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-k2wbnb\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009e54a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-k2wbnb\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0009603d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-k2wbnb\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0009e54e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-k2wbnb\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009e54c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-k2wbnb\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009e54a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-k2wbnb\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":19,"failure_pattern_ids":["503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","d24e138f3b84b77b0ea6793407ac1f41343d0658f0c69312eac159e4f4f7eb45"],"matched_occurrences":19,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511694801571840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4347,"pr_state":"closed","pr_sha":"a82c62541bf569a06efb0a6211c6ae9e8a2c01a4","final_merged_sha":"a82c62541bf569a06efb0a6211c6ae9e8a2c01a4","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T11:52:23Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"01aafd5d34092affcd4779cf5caeb0f7f7ab01af9ef6421c2e68be153f7cace1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511694801571840","occurred_at":"2026-03-16T11:52:23Z","failed_at":"e2e","signature_id":"ae5c23f34846e78e42b4f7dfa56b2ec6839216ba8e29ec451bfc51192214cb7c","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00103a220\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-20-kx5x82-jnbcth\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-kx5x82-jnbcth/operationStatuses/08584279346745938075\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-20-kx5x82-jnbcth\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-kx5x82-jnbcth/operationStatuses/08584279346745938075\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00103a200\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-20-kx5x82-jnbcth\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-kx5x82-jnbcth/operationStatuses/08584279346745938075\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list depl...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00103a220\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-20-kx5x82-jnbcth\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-kx5x82-jnbcth/operationStatuses/08584279346745938075\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-20-kx5x82-jnbcth\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-kx5x82-jnbcth/operationStatuses/08584279346745938075\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00103a200\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-20-kx5x82-jnbcth\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-kx5x82-jnbcth/operationStatuses/08584279346745938075\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list depl...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"1bf5b170a59b4f0f16c9dd99d396d51b8685fe84952dfe89066842eb8c810964","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511694801571840","occurred_at":"2026-03-16T11:52:23Z","failed_at":"e2e","signature_id":"b6aa2820e2d5319fd16580a262e7f706069fcdcabb5248a83e67d2f2bb37c139","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:158]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000747480\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-node-limit-cluster-kp57zw\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-kp57zw/operationStatuses/08584279346142577420\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-91/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-fkbb62/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-node-limit-cluster-kp57zw\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-kp57zw/operationStatuses/08584279346142577420\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-91/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-fkbb62/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments ca...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:158]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000747480\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-node-limit-cluster-kp57zw\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-kp57zw/operationStatuses/08584279346142577420\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-91/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-fkbb62/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-node-limit-cluster-kp57zw\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-kp57zw/operationStatuses/08584279346142577420\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-91/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-fkbb62/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments ca...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"2e922285ff24b5dfb52aab83a4132e371a50d120a8bcae253859632094cdbe0e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511694801571840","occurred_at":"2026-03-16T11:52:23Z","failed_at":"e2e","signature_id":"d67e38906dec75c9d847dfe67f08d62844e50143e09f264aa49c7775b18e597e","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:79]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b295c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-negative-tests-cluster-mbz82w\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-negative-tests-cluster-mbz82w/operationStatuses/08584279348304925301\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-vgcgcw/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oeayaalhj7wjy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-negative-tests-cluster-mbz82w\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-negative-tests-cluster-mbz82w/operationStatuses/08584279348304925301\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-vgcgcw/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oeayaalhj7wjy\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:79]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b295c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-negative-tests-cluster-mbz82w\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-negative-tests-cluster-mbz82w/operationStatuses/08584279348304925301\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-vgcgcw/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oeayaalhj7wjy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-negative-tests-cluster-mbz82w\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-negative-tests-cluster-mbz82w/operationStatuses/08584279348304925301\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-vgcgcw/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oeayaalhj7wjy\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"443a4d9ec2a964883f6860bfc1ac01f23052e1b58bb7cb2251c24a3e351c2376","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511694801571840","occurred_at":"2026-03-16T11:52:23Z","failed_at":"e2e","signature_id":"03fdc31d61de10ed831bbd4e702d06a3a8792092f61dce387a002602091147d5","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000d588a0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-jc9kxj: failed waiting for cluster=\"cluster-zstream-4-21-jc9kxj\" in resourcegroup=\"rg-zstream-upgrade-4-21-wvxm8g\" to finish creating: GET https://rp.j1571840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a56758ae-0e75-4522-b7af-560b514e2896\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a56758ae-0e75-4522-b7af-560b514e2896\",\n \"name\": \"a56758ae-0e75-4522-b7af-560b514e2896\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:43:45.183122663Z\",\n \"endTime\": \"2026-03-16T14:44:27.640803003Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-160/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-160/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-21-jc9kxj: failed waiting for cluster=\\\"cluster-zstream-4-21-jc9kxj\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-wvxm8g\\\" to finish creating: GET https://rp.j1571840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a56758ae-0e75-4522-b7af-560b514e2896\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a56758ae-0e75-4522-b7af-560b514e2896\\\",\\n \\\"name\\\": \\\"a56758ae-0e75-4522-b7af-560b514e2896\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:43:45.183122663Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:44:27.640803003Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-160/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-160/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000d58880\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-jc9kxj\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-wvxm8g\\\" to finish creating: GET https://rp.j1571840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a56758ae-0e75-4522-b7af-560b514e2896\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a56758ae-0e75-4522-b7af-560b514e2896\\\",\\n \\\"name\\\": \\\"a56758ae-0e75-4522-b7af-560b514e2896\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:43:45.183122663Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:44:27.640803003Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-160/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-160/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n---------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000d588a0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-jc9kxj: failed waiting for cluster=\"cluster-zstream-4-21-jc9kxj\" in resourcegroup=\"rg-zstream-upgrade-4-21-wvxm8g\" to finish creating: GET https://rp.j1571840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a56758ae-0e75-4522-b7af-560b514e2896\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a56758ae-0e75-4522-b7af-560b514e2896\",\n \"name\": \"a56758ae-0e75-4522-b7af-560b514e2896\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:43:45.183122663Z\",\n \"endTime\": \"2026-03-16T14:44:27.640803003Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-160/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-160/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-21-jc9kxj: failed waiting for cluster=\\\"cluster-zstream-4-21-jc9kxj\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-wvxm8g\\\" to finish creating: GET https://rp.j1571840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a56758ae-0e75-4522-b7af-560b514e2896\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a56758ae-0e75-4522-b7af-560b514e2896\\\",\\n \\\"name\\\": \\\"a56758ae-0e75-4522-b7af-560b514e2896\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:43:45.183122663Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:44:27.640803003Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-160/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-160/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000d58880\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-jc9kxj\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-wvxm8g\\\" to finish creating: GET https://rp.j1571840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a56758ae-0e75-4522-b7af-560b514e2896\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a56758ae-0e75-4522-b7af-560b514e2896\\\",\\n \\\"name\\\": \\\"a56758ae-0e75-4522-b7af-560b514e2896\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:43:45.183122663Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:44:27.640803003Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-160/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-160/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n---------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"59f8b14ea176a1319f121c412f776f9e9763c128ff977c1174015f99fb5e2d33","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511694801571840","occurred_at":"2026-03-16T11:52:23Z","failed_at":"e2e","signature_id":"ecd9ebbd10b7b37b64fee97fcab935ede5fb5a5f2295c6262b9bc1bca25dd513","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:77]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00075e540\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-engineering-hcp-cluster-dskt5n\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-engineering-hcp-cluster-dskt5n/operationStatuses/08584279347938669527\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-m8fxcd/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-5borhjiq7mwd4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-91/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-engineering-hcp-cluster-dskt5n\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-engineering-hcp-cluster-dskt5n/operationStatuses/08584279347938669527\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-m8fxcd/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-5borhjiq7mwd4\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/su...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:77]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00075e540\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-engineering-hcp-cluster-dskt5n\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-engineering-hcp-cluster-dskt5n/operationStatuses/08584279347938669527\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-m8fxcd/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-5borhjiq7mwd4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-91/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-engineering-hcp-cluster-dskt5n\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-engineering-hcp-cluster-dskt5n/operationStatuses/08584279347938669527\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-m8fxcd/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-5borhjiq7mwd4\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/su...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"6a834f67f7a8ba7467130792daf5b36d0a76a73536cfa8777690be90acd70040","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511694801571840","occurred_at":"2026-03-16T11:52:23Z","failed_at":"e2e","signature_id":"6612c1818a7d5cdd1336c93f76f66eb4eea4afb771b3f2c0f6f49e03466fb206","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:242]: Unexpected error:\n \u003c*errors.joinError | 0xc00133e288\u003e: \n context deadline exceeded\n failed to list ClusterOperators: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n {\n errs: [\n \u003ccontext.deadlineExceededError\u003e{},\n \u003c*fmt.wrapError | 0xc001613460\u003e{\n msg: \"failed to list ClusterOperators: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n err: \u003c*fmt.wrapError | 0xc001613440\u003e{\n msg: \"client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n err: \u003c*errors.errorString | 0xc001584ce0\u003e{\n s: \"rate: Wait(n=1) would exceed context deadline\",\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:242]: Unexpected error:\n \u003c*errors.joinError | 0xc00133e288\u003e: \n context deadline exceeded\n failed to list ClusterOperators: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n {\n errs: [\n \u003ccontext.deadlineExceededError\u003e{},\n \u003c*fmt.wrapError | 0xc001613460\u003e{\n msg: \"failed to list ClusterOperators: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n err: \u003c*fmt.wrapError | 0xc001613440\u003e{\n msg: \"client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n err: \u003c*errors.errorString | 0xc001584ce0\u003e{\n s: \"rate: Wait(n=1) would exceed context deadline\",\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1af131707674faa81fde3d721a8f7cd0dcfd7cac9b068ba9cf8282fe90f6cbc2","failure_pattern":"failed to list ClusterOperators: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline","search_query":"failed to list ClusterOperators: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline"}},{"row_id":"721de19170d1c2f48ce050c688f285e1d7dc697600f50bff83d1aff84a0703e6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511694801571840","occurred_at":"2026-03-16T11:52:23Z","failed_at":"e2e","signature_id":"d97a6b6356ebccc9c8843d3b7bb1c2513c1366306643f11aaa83e566c927ba59","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:86]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000762460\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-vd8tln\" to finish creating: GET https://rp.j1571840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7490142-2b7c-41f4-8adb-9053aa528f9b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7490142-2b7c-41f4-8adb-9053aa528f9b\",\n \"name\": \"f7490142-2b7c-41f4-8adb-9053aa528f9b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:47:50.22465609Z\",\n \"endTime\": \"2026-03-16T14:49:01.889796339Z\",\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"Cluster provisioning failed due to multiple errors\",\n \"details\": [\n {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-91/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-91/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n },\n {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-vd8tln/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-vd8tln\\\" to finish creating: GET https://rp.j1571840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7490142-2b7c-41f4-8adb-9053aa528f9b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7490142-2b7c-41f4-8adb-9053aa528f9b\\\",\\n \\\"name\\\": \\\"f7490142-2b7c-41f4-8adb-9053aa528f9b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:47:50.22465609Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:49:01.889796339Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-91/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-91/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n },\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-vd8tln/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0007622a0\u003e{\n msg: \"failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-vd8tln\\\" to finish creating: GET https://rp.j1571840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7490142-2b7c-41f4-8adb-9053aa528f9b\\n--------------------------------------------------------------------------------\\nRESP...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:86]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000762460\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-vd8tln\" to finish creating: GET https://rp.j1571840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7490142-2b7c-41f4-8adb-9053aa528f9b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7490142-2b7c-41f4-8adb-9053aa528f9b\",\n \"name\": \"f7490142-2b7c-41f4-8adb-9053aa528f9b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:47:50.22465609Z\",\n \"endTime\": \"2026-03-16T14:49:01.889796339Z\",\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"Cluster provisioning failed due to multiple errors\",\n \"details\": [\n {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-91/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-91/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n },\n {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-vd8tln/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-vd8tln\\\" to finish creating: GET https://rp.j1571840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7490142-2b7c-41f4-8adb-9053aa528f9b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7490142-2b7c-41f4-8adb-9053aa528f9b\\\",\\n \\\"name\\\": \\\"f7490142-2b7c-41f4-8adb-9053aa528f9b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:47:50.22465609Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:49:01.889796339Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-91/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-91/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n },\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-vd8tln/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0007622a0\u003e{\n msg: \"failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-vd8tln\\\" to finish creating: GET https://rp.j1571840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7490142-2b7c-41f4-8adb-9053aa528f9b\\n--------------------------------------------------------------------------------\\nRESP...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0aa9ed5d94e7167a12aa30282eff19539fc3fc473772cb96b223d420df66775f","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"b4f995c1d46207e5fd23f3f2324b8ab44b8cf778e0895a5e61b0df22328259b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511694801571840","occurred_at":"2026-03-16T11:52:23Z","failed_at":"e2e","signature_id":"989892ec2ee29acd66db5e180460ca96a5aa2a5f2822603316c64714bdcb8149","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000dd86c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listg9z744\" in resourcegroup=\"ea-list-fsnkgk\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00080f760\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listg9z744\\\" in resourcegroup=\\\"ea-list-fsnkgk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00080f740\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listg9z744\\\" in resourcegroup=\\\"ea-list-fsnkgk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00080f6e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listg9z744\\\" in resourcegroup=\\\"ea-list-fsnkgk\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000dd86c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listg9z744\" in resourcegroup=\"ea-list-fsnkgk\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00080f760\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listg9z744\\\" in resourcegroup=\\\"ea-list-fsnkgk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00080f740\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listg9z744\\\" in resourcegroup=\\\"ea-list-fsnkgk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00080f6e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listg9z744\\\" in resourcegroup=\\\"ea-list-fsnkgk\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"d5425ffb31c8e7c8c76e94ae98daf5b75fae90853440a8470ff673f057e0262a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511694801571840","occurred_at":"2026-03-16T11:52:23Z","failed_at":"e2e","signature_id":"a1e4b71b23cce2070fa0a18a4e4f95876e63f60a63275b4bb7e13ee8bf31995a","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:74]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000804c20\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-autoscale-cluster-jjxpmq\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-autoscale-cluster-jjxpmq/operationStatuses/08584279346744937155\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-nhnjjj/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ssrwqrnhyyabs\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-np-autoscale-cluster-jjxpmq\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-autoscale-cluster-jjxpmq/operationStatuses/08584279346744937155\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-nhnjjj/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ssrwqrnhyyabs\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000804c00\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-np-autoscale-cluster-jjxpmq\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-autoscale-cluster-jjxpmq/operationStatuses/0...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:74]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000804c20\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-autoscale-cluster-jjxpmq\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-autoscale-cluster-jjxpmq/operationStatuses/08584279346744937155\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-nhnjjj/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ssrwqrnhyyabs\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-np-autoscale-cluster-jjxpmq\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-autoscale-cluster-jjxpmq/operationStatuses/08584279346744937155\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-nhnjjj/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ssrwqrnhyyabs\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000804c00\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-np-autoscale-cluster-jjxpmq\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-autoscale-cluster-jjxpmq/operationStatuses/0...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"d54e23de3d171903918a2f14eb1722254e2bbac26235ccec1088b05b4a3ea2a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511694801571840","occurred_at":"2026-03-16T11:52:23Z","failed_at":"e2e","signature_id":"e6f19dc83b868f7f92dcc984ad07b40cb886c070827ffa96c85467b3aa9b20c4","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a80240\u003e: \n failed to create HCP cluster cluster-zstream-4-19-m26nbt: failed waiting for cluster=\"cluster-zstream-4-19-m26nbt\" in resourcegroup=\"rg-zstream-upgrade-4-19-px567h\" to finish creating: GET https://rp.j1571840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17fbc5fc-1a07-4aca-b897-a66cb03b930b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17fbc5fc-1a07-4aca-b897-a66cb03b930b\",\n \"name\": \"17fbc5fc-1a07-4aca-b897-a66cb03b930b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:46:56.283509646Z\",\n \"endTime\": \"2026-03-16T14:48:13.403575067Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-m26nbt: failed waiting for cluster=\\\"cluster-zstream-4-19-m26nbt\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-px567h\\\" to finish creating: GET https://rp.j1571840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17fbc5fc-1a07-4aca-b897-a66cb03b930b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17fbc5fc-1a07-4aca-b897-a66cb03b930b\\\",\\n \\\"name\\\": \\\"17fbc5fc-1a07-4aca-b897-a66cb03b930b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:46:56.283509646Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:48:13.403575067Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a80220\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-19-m26nbt\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-px567h\\\" to finish creating: GET https://rp.j1571840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17fbc5fc-1a07-4aca-b897-a66cb03b930b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17fbc5fc-1a07-4aca-b897-a66cb03b930b\\\",\\n \\\"name\\\": \\\"17fbc5fc-1a07-4aca-b897-a66cb03b930b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:46:56.283509646Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:48:13.403575067Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrot...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a80240\u003e: \n failed to create HCP cluster cluster-zstream-4-19-m26nbt: failed waiting for cluster=\"cluster-zstream-4-19-m26nbt\" in resourcegroup=\"rg-zstream-upgrade-4-19-px567h\" to finish creating: GET https://rp.j1571840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17fbc5fc-1a07-4aca-b897-a66cb03b930b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17fbc5fc-1a07-4aca-b897-a66cb03b930b\",\n \"name\": \"17fbc5fc-1a07-4aca-b897-a66cb03b930b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:46:56.283509646Z\",\n \"endTime\": \"2026-03-16T14:48:13.403575067Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-m26nbt: failed waiting for cluster=\\\"cluster-zstream-4-19-m26nbt\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-px567h\\\" to finish creating: GET https://rp.j1571840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17fbc5fc-1a07-4aca-b897-a66cb03b930b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17fbc5fc-1a07-4aca-b897-a66cb03b930b\\\",\\n \\\"name\\\": \\\"17fbc5fc-1a07-4aca-b897-a66cb03b930b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:46:56.283509646Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:48:13.403575067Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a80220\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-19-m26nbt\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-px567h\\\" to finish creating: GET https://rp.j1571840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17fbc5fc-1a07-4aca-b897-a66cb03b930b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17fbc5fc-1a07-4aca-b897-a66cb03b930b\\\",\\n \\\"name\\\": \\\"17fbc5fc-1a07-4aca-b897-a66cb03b930b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:46:56.283509646Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:48:13.403575067Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrot...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"dbc6c8be980261c4b58b0998b773835a3b9b80f70a4ec0cc6fe300e69f644a59","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511694801571840","occurred_at":"2026-03-16T11:52:23Z","failed_at":"e2e","signature_id":"e0958bf3ada2387b744fd9e999dbff8eb64f8bec2e6404e8e3b09d45bf45d517","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010561e0\u003e: \n failed to cleanup resource group: failed waiting for resourcegroup=\"e2e-cidr-connectivity-hbjgwj\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0011132c0\u003e{\n msg: \"failed to cleanup resource group: failed waiting for resourcegroup=\\\"e2e-cidr-connectivity-hbjgwj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0011132a0\u003e{\n msg: \"failed waiting for resourcegroup=\\\"e2e-cidr-connectivity-hbjgwj\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010561e0\u003e: \n failed to cleanup resource group: failed waiting for resourcegroup=\"e2e-cidr-connectivity-hbjgwj\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0011132c0\u003e{\n msg: \"failed to cleanup resource group: failed waiting for resourcegroup=\\\"e2e-cidr-connectivity-hbjgwj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0011132a0\u003e{\n msg: \"failed waiting for resourcegroup=\\\"e2e-cidr-connectivity-hbjgwj\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d55c4a29bbbf19a43335bfa2157018ebc82a1c8fe8cbea22f78ef2ca4c570f1a","failure_pattern":"failed to cleanup resource group: failed waiting for resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: failed waiting for resourcegroup=\"autoscaling-cluster-2bvkgc\" to finish deleting: context canceled"}},{"row_id":"f5790d403231bce19bb8164d6fa75d9ad6fe7a8b8dbf20ddeadd81a1a76b4148","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511694801571840","occurred_at":"2026-03-16T11:52:23Z","failed_at":"e2e","signature_id":"fa5dbfa7cb7ca2545693627cba3746694b557d622d3a04691779134062a21fda","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001858228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-ppwm4q\" in resourcegroup=\"admin-credential-lifecycle-test-4qglmk\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000b876a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-ppwm4q\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-4qglmk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b87660\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-ppwm4q\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-4qglmk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b87640\u003e{\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-ppwm4q\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-4qglmk\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001858228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-ppwm4q\" in resourcegroup=\"admin-credential-lifecycle-test-4qglmk\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000b876a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-ppwm4q\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-4qglmk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b87660\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-ppwm4q\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-4qglmk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b87640\u003e{\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-ppwm4q\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-4qglmk\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["0aa9ed5d94e7167a12aa30282eff19539fc3fc473772cb96b223d420df66775f","1af131707674faa81fde3d721a8f7cd0dcfd7cac9b068ba9cf8282fe90f6cbc2","503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","d55c4a29bbbf19a43335bfa2157018ebc82a1c8fe8cbea22f78ef2ca4c570f1a"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4351/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511496104808448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4351,"pr_state":"closed","pr_sha":"eac14dfe2d1e1037f83e2effb8b3fabd24958599","final_merged_sha":"131806c6037c274a80c20df0a02233247a78c57e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:51:36Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"307d55a673387612df09562d7682ade6959852d60472cc0d01e69ed5c07b7fbb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4351/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511496104808448","occurred_at":"2026-03-16T11:51:36Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-16T14:04:14.431Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-16T14:04:16.944Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-16T14:04:18.414Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=1525bda5783b284829b029d62126cb30fcf582c6f2a60075236dde3ced9a8150\ntime=2026-03-16T14:05:48.773Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j4808448/providers/Microsoft.Resources/deployments/1525bda5783b284829b029d62126cb30fcf582c6f2a60075236dde3ced9a8150/operationStatuses/08584279370280120648\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4808448/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4808448/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j4808448\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 64a663a4-5838-466b-8471-066c963f3378:3/16/2026 2:05:37 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4355/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511293431844864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4355,"pr_state":"closed","pr_sha":"0e391072d998cf7799f9dcd335b611eb00e15df2","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:50:47Z"},"failed_at":["e2e"],"failed_test_count":9,"occurrences":[{"row_id":"087a2a12c101023184d1ca2c8fb1f2622fd7c3f7ea81ce8e966be0c653bac514","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4355/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511293431844864","occurred_at":"2026-03-16T11:50:47Z","failed_at":"e2e","signature_id":"7fd9e834d0cf89b58ccef9ad24056078195255b9502a942a4a7b710f857db084","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.004s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0009ce3c0\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.004s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0009ce3c0\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}},{"row_id":"4f61ed2eec4959577a323f83bc4eb53eee158eaa4d095c8cbd89a5d643292670","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4355/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511293431844864","occurred_at":"2026-03-16T11:50:47Z","failed_at":"e2e","signature_id":"2e0418f5aee27498fa4844ec8b91faad8fa4b0d3c4e1571058afb1c5b9cdbd09","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0016161b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-rpr688\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000ba3680\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-rpr688\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ba3640\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-rpr688\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ba3620\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-rpr688\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0016161b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-rpr688\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000ba3680\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-rpr688\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ba3640\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-rpr688\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ba3620\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-rpr688\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"500213432f1e88e142add45d14b7eac1ed24c7c523ab6fe0c86bbd65554464d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4355/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511293431844864","occurred_at":"2026-03-16T11:50:47Z","failed_at":"e2e","signature_id":"44e4587a4e172f03f2c73c033a52fdcae49b72a99a8b657e53050db0626f9331","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000896c80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-20-9btg4b-t58z2t\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-9btg4b-t58z2t/operationStatuses/08584279346703677235\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-mkw9kr/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-az4uc6ctjgcpo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-249/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-20-9btg4b-t58z2t\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-9btg4b-t58z2t/operationStatuses/08584279346703677235\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-mkw9kr/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-az4uc6ctjgcpo\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-249/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignme...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000896c80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-20-9btg4b-t58z2t\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-9btg4b-t58z2t/operationStatuses/08584279346703677235\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-mkw9kr/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-az4uc6ctjgcpo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-249/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-20-9btg4b-t58z2t\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-9btg4b-t58z2t/operationStatuses/08584279346703677235\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-mkw9kr/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-az4uc6ctjgcpo\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-249/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignme...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"5e3c9b6af06734b4e56f42f075eb4a862ed2338e7a61d231b8c9243620740655","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4355/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511293431844864","occurred_at":"2026-03-16T11:50:47Z","failed_at":"e2e","signature_id":"6b28ac937257fc8e87c69847b762c8bcf31787129a7524c621350950d151b8ed","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0012d3758\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-cfx5gd\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000dee6e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-cfx5gd\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000dee6c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-cfx5gd\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000dee6a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-cfx5gd\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0012d3758\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-cfx5gd\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000dee6e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-cfx5gd\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000dee6c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-cfx5gd\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000dee6a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-cfx5gd\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"664696d1d72aabc4b18a3e7e4cc8ba682a6147d7118b57567cf164f290ea27ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4355/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511293431844864","occurred_at":"2026-03-16T11:50:47Z","failed_at":"e2e","signature_id":"354106ffe61a6be1d71d06c160155cce5ce9598a75aa10297b539ae8607f0638","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010424b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-fvpr9k\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007b1c80\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-fvpr9k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007b1c60\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-fvpr9k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007b1c40\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-fvpr9k\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010424b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-fvpr9k\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007b1c80\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-fvpr9k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007b1c60\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-fvpr9k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007b1c40\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-fvpr9k\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"a86274aafbe4c6750d8ed0340a5cdd89d0da4aca71c30827938eb2433c89b060","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4355/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511293431844864","occurred_at":"2026-03-16T11:50:47Z","failed_at":"e2e","signature_id":"6329c77e713e26184cc46aca5b5ea77d6fe467c0f01781c4160379ba8088d6e5","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e70258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-qxcp4w\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a11380\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-qxcp4w\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a11360\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-qxcp4w\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a11340\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-qxcp4w\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e70258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-qxcp4w\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a11380\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-qxcp4w\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a11360\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-qxcp4w\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a11340\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-qxcp4w\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"be27d3f486b4ce414b2db75bf6c7bb7a098bcb61c66dae4635769b7a2274a750","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4355/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511293431844864","occurred_at":"2026-03-16T11:50:47Z","failed_at":"e2e","signature_id":"89b1e870171d73c00d40bfdd6e0cb7642ccee5f4b676268e310acb5e7a3acee6","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0009f8600\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-jff46k\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00100b960\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-jff46k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00100b940\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-jff46k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00100b920\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-jff46k\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0009f8600\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-jff46k\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00100b960\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-jff46k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00100b940\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-jff46k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00100b920\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-jff46k\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"ca66eb1a5b9645d84faf4c9fdab9ddce32fac1b164380ab91de6da2e8162af02","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4355/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511293431844864","occurred_at":"2026-03-16T11:50:47Z","failed_at":"e2e","signature_id":"810d7ffd226c9238baed9ccf34b49fa0cd6258c1c7d638c5327e867f0d70ef5e","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b184c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-21-tf444c-599hwd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-tf444c-599hwd/operationStatuses/08584279346098647612\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-196/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-gk2sxr/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-imkpyhgupivhu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-21-tf444c-599hwd\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-tf444c-599hwd/operationStatuses/08584279346098647612\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-196/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-gk2sxr/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-imkpyhgupivhu\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b184c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-21-tf444c-599hwd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-tf444c-599hwd/operationStatuses/08584279346098647612\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-196/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-gk2sxr/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-imkpyhgupivhu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-21-tf444c-599hwd\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-tf444c-599hwd/operationStatuses/08584279346098647612\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-196/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-gk2sxr/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-imkpyhgupivhu\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"f411cdf8e7920d6731ea9cf7ad004fe0437b3f9e67aa84eef8652d6406753c90","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4355/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033511293431844864","occurred_at":"2026-03-16T11:50:47Z","failed_at":"e2e","signature_id":"ac0c22c4e18ca7ce07d18e447c35533b3855f8d1061692a6207e33019443edcc","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000cd02a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-25sdtx\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000761960\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-25sdtx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000761940\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-25sdtx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000761920\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-25sdtx\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000cd02a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-25sdtx\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000761960\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-25sdtx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000761940\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-25sdtx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000761920\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-25sdtx\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":9,"failure_pattern_ids":["567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d"],"matched_occurrences":9,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510839050309632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4364,"pr_state":"open","pr_sha":"2e6c30457cd0d9389f2bee67f73db6e237bed604","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:48:59Z"},"failed_at":["e2e"],"failed_test_count":20,"occurrences":[{"row_id":"05160d265c9b4f121dd2e80635b40d7954af0f4d120a5e708a46a9535f29c82d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510839050309632","occurred_at":"2026-03-16T11:48:59Z","failed_at":"e2e","signature_id":"07e6f57e3d8d2dcb5c05eab2c9f38ff179dfed53ee55ac2d7d9294998e59f706","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:82]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0008873e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-admin-cred-lifecycle-g7fgt6-79nm6m\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-admin-cred-lifecycle-g7fgt6-79nm6m/operationStatuses/08584279347852100320\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-162/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-8prhlz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-jjxzjxtwa342w\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-admin-cred-lifecycle-g7fgt6-79nm6m\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-admin-cred-lifecycle-g7fgt6-79nm6m/operationStatuses/08584279347852100320\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-162/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-8prhlz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-jjxzjxtwa342w\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:82]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0008873e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-admin-cred-lifecycle-g7fgt6-79nm6m\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-admin-cred-lifecycle-g7fgt6-79nm6m/operationStatuses/08584279347852100320\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-162/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-8prhlz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-jjxzjxtwa342w\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-admin-cred-lifecycle-g7fgt6-79nm6m\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-admin-cred-lifecycle-g7fgt6-79nm6m/operationStatuses/08584279347852100320\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-162/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-8prhlz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-jjxzjxtwa342w\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"0742be9a0372dd94f5dd41a3499e769b15d21ecc56d9c5409bcc891ff650fdc4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510839050309632","occurred_at":"2026-03-16T11:48:59Z","failed_at":"e2e","signature_id":"c6a9c07ae2482ad7eea650e74353457ce924fb6e07322c7ad30a3a61d3720ef2","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:79]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0008ab440\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-disabled-image-registry-hcp-cluster-9fgcjk\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-disabled-image-registry-hcp-cluster-9fgcjk/operationStatuses/08584279346675224659\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-qfvgcd/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-qmh74zogaodhg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-disabled-image-registry-hcp-cluster-9fgcjk\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-disabled-image-registry-hcp-cluster-9fgcjk/operationStatuses/08584279346675224659\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-qfvgcd/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-qmh74zogaodhg\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0008ab420\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-disabled-image-registry-hcp-cluster-9fgcjk\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployme...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:79]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0008ab440\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-disabled-image-registry-hcp-cluster-9fgcjk\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-disabled-image-registry-hcp-cluster-9fgcjk/operationStatuses/08584279346675224659\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-qfvgcd/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-qmh74zogaodhg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-disabled-image-registry-hcp-cluster-9fgcjk\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-disabled-image-registry-hcp-cluster-9fgcjk/operationStatuses/08584279346675224659\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-qfvgcd/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-qmh74zogaodhg\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0008ab420\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-disabled-image-registry-hcp-cluster-9fgcjk\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployme...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"29a6d8fabdf686efc5b898dfc298ccd2642d95a7bd69cb8e2b185952defac80b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510839050309632","occurred_at":"2026-03-16T11:48:59Z","failed_at":"e2e","signature_id":"bed3d04c428af260f9aaa8903d50cb6178a997980da1aa46f119ad963895e173","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00099c8a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5kt9r5\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000fb7380\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-5kt9r5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000fb7360\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-5kt9r5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000fb7340\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-5kt9r5\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00099c8a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5kt9r5\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000fb7380\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-5kt9r5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000fb7360\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-5kt9r5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000fb7340\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-5kt9r5\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"311b2a88793cdc1a40d5e72dd56dfa45a4e0b0faf668160d22857a39472cce82","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510839050309632","occurred_at":"2026-03-16T11:48:59Z","failed_at":"e2e","signature_id":"defd68f4c0dc214d67d231affc2e919e3521501b8c3b9a128b0f07b8b6611a32","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.007s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000917b70\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.007s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000917b70\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}},{"row_id":"360bdc560524ac1ace16be86502e2a7bb5371562ec74062edafb05b187431c9d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510839050309632","occurred_at":"2026-03-16T11:48:59Z","failed_at":"e2e","signature_id":"52dcc15434758e6f1746c8c057d9b68d63ba57b2e620873a72738adee9c633b2","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010ae0d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-qxpbrx\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000d76c60\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-qxpbrx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d76c40\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-qxpbrx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d76c20\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-qxpbrx\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010ae0d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-qxpbrx\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000d76c60\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-qxpbrx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d76c40\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-qxpbrx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d76c20\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-qxpbrx\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"5019db950be5d98a3fcf6acd22e29ae284ef675a4033fbbc2bd0695bcbff4a6a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510839050309632","occurred_at":"2026-03-16T11:48:59Z","failed_at":"e2e","signature_id":"8e241205ee8671d3b08a700976873cb4423f4c348b2d89a03bfc3091ea2489ad","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000f11820\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-21-d4mcm7-gjd5hv\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-d4mcm7-gjd5hv/operationStatuses/08584279347271369558\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-195/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-21-d4mcm7-gjd5hv\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-d4mcm7-gjd5hv/operationStatuses/08584279347271369558\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-195/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000f117e0\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-21-d4mcm7-gjd5hv\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-d4mcm7-gjd5hv/operationStatuses/08584279347271369558\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-195/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000f11820\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-21-d4mcm7-gjd5hv\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-d4mcm7-gjd5hv/operationStatuses/08584279347271369558\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-195/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-21-d4mcm7-gjd5hv\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-d4mcm7-gjd5hv/operationStatuses/08584279347271369558\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-195/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000f117e0\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-21-d4mcm7-gjd5hv\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-d4mcm7-gjd5hv/operationStatuses/08584279347271369558\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-195/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"57d6ff2dce4188537678bd95ee4468261eb424709ccd67df51e51294956d7ba5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510839050309632","occurred_at":"2026-03-16T11:48:59Z","failed_at":"e2e","signature_id":"41db552db5f5c1097830a4ee92ef8abe26205836acbf9c072d3ba95c37c83b58","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000eab398\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-csfq6k\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00105cf00\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-csfq6k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00105cee0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-csfq6k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00105cec0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-csfq6k\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000eab398\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-csfq6k\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00105cf00\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-csfq6k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00105cee0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-csfq6k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00105cec0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-csfq6k\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"58168f72c8bb6dac619b720bfecda2e8afc0d366d0cce0045bc2d82c801be35d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510839050309632","occurred_at":"2026-03-16T11:48:59Z","failed_at":"e2e","signature_id":"f1884d2bc6d5bdfa951745f6f4b642ee537a37898c737143cd0fe0fdbfdf89e0","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001784240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-vxfmp6\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0008b35a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-vxfmp6\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008b3580\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-vxfmp6\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008b3560\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-vxfmp6\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001784240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-vxfmp6\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0008b35a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-vxfmp6\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008b3580\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-vxfmp6\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008b3560\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-vxfmp6\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"5b4b52279d88486b4c2b3f88dcac565097dc555cd74d6caf671b6f5d220e09f0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510839050309632","occurred_at":"2026-03-16T11:48:59Z","failed_at":"e2e","signature_id":"cf87a3d8b3acc82bdb105f3709247d7087f46afd29241f49441064b803c23a5e","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00187a1b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-q5hmwx\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000729080\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-q5hmwx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000729040\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-q5hmwx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000729000\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-q5hmwx\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00187a1b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-q5hmwx\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000729080\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-q5hmwx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000729040\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-q5hmwx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000729000\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-q5hmwx\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"5c478a3cd073967d95e05c542d98428d10a995327c0dbf005989cdb09de2b1e6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510839050309632","occurred_at":"2026-03-16T11:48:59Z","failed_at":"e2e","signature_id":"f6c348e5d59bd809ea8005bed55f2bedf1f533f7e853ed3caa0cd34578677c3c","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0014521c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-pwhwps\" in resourcegroup=\"rg-zstream-upgrade-4-20-kf4b25\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a4f5c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-pwhwps\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-kf4b25\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a4f5a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-pwhwps\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-kf4b25\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a4f580\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-20-pwhwps\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-kf4b25\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0014521c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-pwhwps\" in resourcegroup=\"rg-zstream-upgrade-4-20-kf4b25\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a4f5c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-pwhwps\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-kf4b25\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a4f5a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-pwhwps\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-kf4b25\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a4f580\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-20-pwhwps\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-kf4b25\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"6e9b2ffeeb2a2ba4ca959c5fe899a59f9c0f615f9c68316e64a464c7ae239160","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510839050309632","occurred_at":"2026-03-16T11:48:59Z","failed_at":"e2e","signature_id":"f86190c12a5893402698e763f7e441f8cf08354b03e051cf228e189a0f9609ef","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:73]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000f1b940\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-update-nodes-hcp-cluster-dc7gqr\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-dc7gqr/operationStatuses/08584279346155999693\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-162/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-4nkcvc/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-np-update-nodes-hcp-cluster-dc7gqr\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-dc7gqr/operationStatuses/08584279346155999693\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-162/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-4nkcvc/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:73]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000f1b940\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-update-nodes-hcp-cluster-dc7gqr\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-dc7gqr/operationStatuses/08584279346155999693\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-162/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-4nkcvc/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-np-update-nodes-hcp-cluster-dc7gqr\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-dc7gqr/operationStatuses/08584279346155999693\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-162/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-4nkcvc/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"786a684d272b97d7044e7e30a156b460e71f9cbc5ba3f6d4c23d0ddcc1b5f909","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510839050309632","occurred_at":"2026-03-16T11:48:59Z","failed_at":"e2e","signature_id":"97c3e3a1b33ef343091d5519958b81b4e1c40dd46fb37508ad038254799f1a2b","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:158]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001522cc0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-node-limit-cluster-zctskf\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-zctskf/operationStatuses/08584279348466930659\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-280/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8mrvmw/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-node-limit-cluster-zctskf\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-zctskf/operationStatuses/08584279348466930659\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-280/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8mrvmw/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments c...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:158]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001522cc0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-node-limit-cluster-zctskf\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-zctskf/operationStatuses/08584279348466930659\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-280/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8mrvmw/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-node-limit-cluster-zctskf\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-zctskf/operationStatuses/08584279348466930659\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-280/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8mrvmw/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments c...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"858f5b56085471247e55dc7af97b0f1f17caaa4d95d3b90f5fdf3f7637f02b3c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510839050309632","occurred_at":"2026-03-16T11:48:59Z","failed_at":"e2e","signature_id":"1800e82548093124b268c7e465f7c5b41dfd2f3696bed10b5f42d37dbccf3dff","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0009e4aa0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-19-jrlm9l-9f77ll\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-19-jrlm9l-9f77ll/operationStatuses/08584279346044663550\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-cznbgf/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-kue5o72yqqisy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-175/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-19-jrlm9l-9f77ll\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-19-jrlm9l-9f77ll/operationStatuses/08584279346044663550\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-cznbgf/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-kue5o72yqqisy\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0009e4aa0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-19-jrlm9l-9f77ll\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-19-jrlm9l-9f77ll/operationStatuses/08584279346044663550\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-cznbgf/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-kue5o72yqqisy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-175/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-19-jrlm9l-9f77ll\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-19-jrlm9l-9f77ll/operationStatuses/08584279346044663550\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-cznbgf/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-kue5o72yqqisy\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"8e503bea93872c3e0dc9e4419de236b9da2274d9a49ef6515a1ec0fd73a18c4c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510839050309632","occurred_at":"2026-03-16T11:48:59Z","failed_at":"e2e","signature_id":"8bf647f25ad88cc798dc3740a8d8c0d5352d0246593911c84b3720a48132e1ce","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001066318\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-m96cdz\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007e5280\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-m96cdz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007e5240\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-m96cdz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007e5220\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-m96cdz\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001066318\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-m96cdz\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007e5280\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-m96cdz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007e5240\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-m96cdz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007e5220\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-m96cdz\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"91aee2f43cd1155f4b3f3fd86492bf80b6a082adcc0cb4073fc0bf0ac6838714","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510839050309632","occurred_at":"2026-03-16T11:48:59Z","failed_at":"e2e","signature_id":"83cebf028b9d55eb6dbc51c8e5b27d3e29f42a941da248ad25afb71c20a65e4d","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0004e55a0\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-tblwrw\" to finish creating: GET https://rp.j0309632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f9c4e00-f5e2-4c0e-bb9e-7c76759fe905\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f9c4e00-f5e2-4c0e-bb9e-7c76759fe905\",\n \"name\": \"8f9c4e00-f5e2-4c0e-bb9e-7c76759fe905\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:37:56.629756101Z\",\n \"endTime\": \"2026-03-16T14:38:48.032996565Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-tblwrw/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '2' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-tblwrw\\\" to finish creating: GET https://rp.j0309632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f9c4e00-f5e2-4c0e-bb9e-7c76759fe905\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f9c4e00-f5e2-4c0e-bb9e-7c76759fe905\\\",\\n \\\"name\\\": \\\"8f9c4e00-f5e2-4c0e-bb9e-7c76759fe905\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:37:56.629756101Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:38:48.032996565Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-tblwrw/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '2' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0004e5560\u003e{\n msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-tblwrw\\\" to finish creating: GET https://rp.j0309632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f9c4e00-f5e2-4c0e-bb9e-7c76759fe905\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f9c4e00-f5e2-4c0e-bb9e-7c76759fe905\\\",\\n \\\"name\\\": \\\"8f9c4e00-f5e2-4c0e-bb9e-7c76759fe905\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:37:56.629756101Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:38:48.032996565Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-tblwrw/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again aft...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0004e55a0\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-tblwrw\" to finish creating: GET https://rp.j0309632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f9c4e00-f5e2-4c0e-bb9e-7c76759fe905\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f9c4e00-f5e2-4c0e-bb9e-7c76759fe905\",\n \"name\": \"8f9c4e00-f5e2-4c0e-bb9e-7c76759fe905\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:37:56.629756101Z\",\n \"endTime\": \"2026-03-16T14:38:48.032996565Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-tblwrw/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '2' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-tblwrw\\\" to finish creating: GET https://rp.j0309632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f9c4e00-f5e2-4c0e-bb9e-7c76759fe905\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f9c4e00-f5e2-4c0e-bb9e-7c76759fe905\\\",\\n \\\"name\\\": \\\"8f9c4e00-f5e2-4c0e-bb9e-7c76759fe905\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:37:56.629756101Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:38:48.032996565Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-tblwrw/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '2' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0004e5560\u003e{\n msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-tblwrw\\\" to finish creating: GET https://rp.j0309632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f9c4e00-f5e2-4c0e-bb9e-7c76759fe905\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f9c4e00-f5e2-4c0e-bb9e-7c76759fe905\\\",\\n \\\"name\\\": \\\"8f9c4e00-f5e2-4c0e-bb9e-7c76759fe905\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:37:56.629756101Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:38:48.032996565Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-tblwrw/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again aft...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b5b4090dda95941c15f59f203d24b23d3d00ede2c63b3dfa48e876a2dbfb4848","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510839050309632","occurred_at":"2026-03-16T11:48:59Z","failed_at":"e2e","signature_id":"8169f52f50026ee490fe08696abb3c605ac9733523161d8a1752b90b7423d0a9","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:79]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000d92820\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-negative-tests-cluster-rfdqjk\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-negative-tests-cluster-rfdqjk/operationStatuses/08584279348903194143\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-dcdfsp/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oeayaalhj7wjy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-195/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-negative-tests-cluster-rfdqjk\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-negative-tests-cluster-rfdqjk/operationStatuses/08584279348903194143\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-dcdfsp/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oeayaalhj7wjy\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:79]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000d92820\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-negative-tests-cluster-rfdqjk\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-negative-tests-cluster-rfdqjk/operationStatuses/08584279348903194143\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-dcdfsp/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oeayaalhj7wjy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-195/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-negative-tests-cluster-rfdqjk\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-negative-tests-cluster-rfdqjk/operationStatuses/08584279348903194143\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-dcdfsp/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oeayaalhj7wjy\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"cb111c17ca391a4bec4d3cb5126adaf15921dea89d1ad0768a70a0d43078a3bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510839050309632","occurred_at":"2026-03-16T11:48:59Z","failed_at":"e2e","signature_id":"f304fd990958fa6537a48911d7bcd2a64ab8979e367a99bc3d49ceb930b4b80d","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:150]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e21a80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-patch-tags-cluster-zdjpzc\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-patch-tags-cluster-zdjpzc/operationStatuses/08584279347877156231\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-xxn4s8/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fklo75htpxzxc\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-patch-tags-cluster-zdjpzc\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-patch-tags-cluster-zdjpzc/operationStatuses/08584279347877156231\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-xxn4s8/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fklo75htpxzxc\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e21a60\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-patch-tags-cluster-zdjpzc\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-patch-tags-cluster-zdjpzc/operationStatuses/08584279347877156231\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation fa...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:150]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e21a80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-patch-tags-cluster-zdjpzc\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-patch-tags-cluster-zdjpzc/operationStatuses/08584279347877156231\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-xxn4s8/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fklo75htpxzxc\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-patch-tags-cluster-zdjpzc\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-patch-tags-cluster-zdjpzc/operationStatuses/08584279347877156231\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-xxn4s8/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fklo75htpxzxc\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e21a60\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-patch-tags-cluster-zdjpzc\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-patch-tags-cluster-zdjpzc/operationStatuses/08584279347877156231\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation fa...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"eaa5dafc773953d664d7aecd586e9d8ddf17a4a9a5fbf43374a0f873e26fe4a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510839050309632","occurred_at":"2026-03-16T11:48:59Z","failed_at":"e2e","signature_id":"53979142c06df45e837105896e8972060dade2a31bd6f046b9906c3743bca9bb","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f901c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-qxdqml\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00059d900\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-qxdqml\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00059d8e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-qxdqml\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00059d8c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-qxdqml\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f901c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-qxdqml\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00059d900\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-qxdqml\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00059d8e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-qxdqml\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00059d8c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-qxdqml\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"f6a738145ad174072cd83a1443eff112897d2828b762b22397de36f3ae0efd56","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510839050309632","occurred_at":"2026-03-16T11:48:59Z","failed_at":"e2e","signature_id":"6284b7418d561a8d5091b4ccc88e90f3c25cc7420d13fda03655d8a7c1b645b6","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0014a8198\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listf78f8t\" in resourcegroup=\"ea-list-prmcsh\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000c238c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listf78f8t\\\" in resourcegroup=\\\"ea-list-prmcsh\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000c238a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listf78f8t\\\" in resourcegroup=\\\"ea-list-prmcsh\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000c23880\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listf78f8t\\\" in resourcegroup=\\\"ea-list-prmcsh\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0014a8198\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listf78f8t\" in resourcegroup=\"ea-list-prmcsh\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000c238c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listf78f8t\\\" in resourcegroup=\\\"ea-list-prmcsh\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000c238a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listf78f8t\\\" in resourcegroup=\\\"ea-list-prmcsh\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000c23880\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listf78f8t\\\" in resourcegroup=\\\"ea-list-prmcsh\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"ffeb0f9bf12fe3be2c947d99727138e0af8052388d140ae1a0c1dfa91e68f9a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510839050309632","occurred_at":"2026-03-16T11:48:59Z","failed_at":"e2e","signature_id":"973fec819efe52c0d06eba0149e6766623a8d36d67eb70897f1f0d30c4eca045","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0013f4330\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-mqg6sm\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000eb1740\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-mqg6sm\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000eb1720\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-mqg6sm\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000eb1700\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-mqg6sm\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0013f4330\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-mqg6sm\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000eb1740\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-mqg6sm\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000eb1720\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-mqg6sm\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000eb1700\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-mqg6sm\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":20,"failure_pattern_ids":["5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d"],"matched_occurrences":20,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4369/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510636851302400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4369,"pr_state":"open","pr_sha":"ecf4a8cd2207eaef930c24d9ea87a14f55550f3d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:48:11Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"1ae83634c3791490231b711d963ed01aa7d8cdbbeafe72dd6f9008909369bfc7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4369/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510636851302400","occurred_at":"2026-03-16T11:48:11Z","failed_at":"e2e","signature_id":"e90272f51226ae64c333b2695286369a33412256b71b089d3cf032a6ca1ee469","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"382910 pc=0x2b98a38\ncrypto/tls.(*Conn).readFromUntil(0xc000806e08, {0x81cbfc0, 0xc000d38128}, 0xc0003829d0?)\n\t/usr/local/go/src/crypto/tls/conn.go:841 +0xde fp=0xc0003829a0 sp=0xc000382968 pc=0x2d6e89e\ncrypto/tls.(*Conn).readRecordOrCCS(0xc000806e08, 0x0)\n\t/usr/local/go/src/crypto/tls/conn.go:630 +0x3db fp=0xc000382c18 sp=0xc0003829a0 pc=0x2d6b91b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/local/go/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc000806e08, {0xc00110f000, 0x1000, 0x31b05e0?})\n\t/usr/local/go/src/crypto/tls/conn.go:1397 +0x145 fp=0xc000382c80 sp=0xc000382c18 pc=0x2d72205\nbufio.(*Reader).Read(0xc00022cfc0, {0xc00061e2e4, 0x9, 0x31be72e?})\n\t/usr/local/go/src/bufio/bufio.go:245 +0x197 fp=0xc000382cb8 sp=0xc000382c80 pc=0x2c341f7\nio.ReadAtLeast({0x81caba0, 0xc00022cfc0}, {0xc00061e2e4, 0x9, 0x9}, 0x9)\n\t/usr/local/go/src/io/io.go:335 +0x8e fp=0xc000382d00 sp=0xc000382cb8 pc=0x2b28aee\nio.ReadFull(...)\n\t/usr/local/go/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc00061e2e4, 0x9, 0xc000000507?}, {0x81caba0?, 0xc00022cfc0?})\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/frame.go:242 +0x65 fp=0xc000382d50 sp=0xc000382d00 pc=0x31a7d85\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc00061e2a0)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/frame.go:505 +0x6b fp=0xc000382dd8 sp=0xc000382d50 pc=0x31a894b\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc00061e2a0)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/frame.go:564 +0x18 fp=0xc000382e18 sp=0xc000382dd8 pc=0x31a90b8\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000382fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:2258 +0xca fp=0xc000382f60 sp=0xc000382e18 pc=0x31bb66a\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc001688540)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:2127 +0x52 fp=0xc000382fc8 sp=0xc000382f60 pc=0x31ba9f2\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:880 +0x25 fp=0xc000382fe0 sp=0xc000382fc8 pc=0x31b4185\nruntime.goexit({})\n\t/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000382fe8 sp=0xc000382fe0 pc=0x2ad6921\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 144\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:880 +0xda5\n\ngoroutine 1383 gp=0xc00161cfc0 m=nil [sync.WaitGroup.Wait, 3 minutes]:\nruntime.gopark(0xc001729af0?, 0x2a657d4?, 0x20?, 0xb9?, 0x7ffb3a4a5d06?)\n\t/usr/local/go/src/runtime/proc.go:460 +0xce fp=0xc000d9ba90 sp=0xc000d9ba70 pc=0x2ace28e\nruntime.goparkunlock(...)\n\t/usr/local/go/src/runtime/proc.go:466\nruntime.semacquire1(0xc0014c56d0, 0x0, 0x1, 0x0, 0x19)\n\t/usr/local/go/src/runtime/sema.go:192 +0x21d fp=0xc000d9baf8 sp=0xc000d9ba90 pc=0x2aaa3dd\nsync.runtime_SemacquireWaitGroup(0xc001004c00?, 0xc0?)\n\t/usr/local/go/src/runtime/sema.go:114 +0x2e fp=0xc000d9bb30 sp=0xc000d9baf8 pc=0x2acf9ae\nsync.(*WaitGroup).Wait(0xc0014c56c8)\n\t/usr/local/go/src/sync/waitgroup.go:206 +0x85 fp=0xc000d9bb58 sp=0xc000d9bb30 pc=0x2af2e05\ngolang.org/x/sync/errgroup.(*Group).Wait(0xc0014c56c0)\n\t/go/pkg/mod/golang.org/x/sync@v0.19.0/errgroup/errgroup.go:56 +0x1e fp=0xc000d9bb78 sp=0xc000d9bb58 pc=0x376d67e\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters({0x7ffb3818b460, 0xc0016fe180}, 0xc001004540, {0xc000a86dc0, 0x1c}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:319 +0x8ef fp=0xc000d9be20 sp=0xc000d9bb78 pc=0x4eb296f\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).cleanupResourceGroup(0xc0008a4d80, {0x7ffb3818b460, 0xc0016fe180}, {0xc000a86dc0, 0x1c}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:433 +0x294 fp=0xc000d9bf28 sp=0xc000d9be20 pc=0x4ec7c74\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.func1({0x7ffb3818b460, 0xc0016fe180})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:294 +0x14d fp=0xc000d9bfc0 sp=0xc000d9bf28 pc=0x4ec67ad\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:303 +0x28 fp=0xc000d9bfe0 sp=0xc000d9bfc0 pc=0x4ec6628\nruntime.goexit({})\n\t/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000d9bfe8 sp=0xc000d9bfe0 pc=0x2ad6921\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 1382\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:287 +0xc5\n\nrax 0xca\nrbx 0x0\nrcx 0x2ad8743\nrdx 0x0\nrdi 0xb361a90\nrsi 0x80\nrbp 0x7ffd9affcdd0\nrsp 0x7ffd9affcd88\nr8 0x0\nr9 0x0\nr10 0x0\nr11 0x286\nr12 0x8b\nr13 0x1\nr14 0xb35f9a0\nr15 0x1\nrip 0x2ad8741\nrflags 0x286\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command\n382910 pc=0x2b98a38\ncrypto/tls.(*Conn).readFromUntil(0xc000806e08, {0x81cbfc0, 0xc000d38128}, 0xc0003829d0?)\n\t/usr/local/go/src/crypto/tls/conn.go:841 +0xde fp=0xc0003829a0 sp=0xc000382968 pc=0x2d6e89e\ncrypto/tls.(*Conn).readRecordOrCCS(0xc000806e08, 0x0)\n\t/usr/local/go/src/crypto/tls/conn.go:630 +0x3db fp=0xc000382c18 sp=0xc0003829a0 pc=0x2d6b91b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/local/go/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc000806e08, {0xc00110f000, 0x1000, 0x31b05e0?})\n\t/usr/local/go/src/crypto/tls/conn.go:1397 +0x145 fp=0xc000382c80 sp=0xc000382c18 pc=0x2d72205\nbufio.(*Reader).Read(0xc00022cfc0, {0xc00061e2e4, 0x9, 0x31be72e?})\n\t/usr/local/go/src/bufio/bufio.go:245 +0x197 fp=0xc000382cb8 sp=0xc000382c80 pc=0x2c341f7\nio.ReadAtLeast({0x81caba0, 0xc00022cfc0}, {0xc00061e2e4, 0x9, 0x9}, 0x9)\n\t/usr/local/go/src/io/io.go:335 +0x8e fp=0xc000382d00 sp=0xc000382cb8 pc=0x2b28aee\nio.ReadFull(...)\n\t/usr/local/go/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc00061e2e4, 0x9, 0xc000000507?}, {0x81caba0?, 0xc00022cfc0?})\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/frame.go:242 +0x65 fp=0xc000382d50 sp=0xc000382d00 pc=0x31a7d85\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc00061e2a0)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/frame.go:505 +0x6b fp=0xc000382dd8 sp=0xc000382d50 pc=0x31a894b\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc00061e2a0)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/frame.go:564 +0x18 fp=0xc000382e18 sp=0xc000382dd8 pc=0x31a90b8\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000382fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:2258 +0xca fp=0xc000382f60 sp=0xc000382e18 pc=0x31bb66a\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc001688540)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:2127 +0x52 fp=0xc000382fc8 sp=0xc000382f60 pc=0x31ba9f2\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:880 +0x25 fp=0xc000382fe0 sp=0xc000382fc8 pc=0x31b4185\nruntime.goexit({})\n\t/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000382fe8 sp=0xc000382fe0 pc=0x2ad6921\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 144\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:880 +0xda5\n\ngoroutine 1383 gp=0xc00161cfc0 m=nil [sync.WaitGroup.Wait, 3 minutes]:\nruntime.gopark(0xc001729af0?, 0x2a657d4?, 0x20?, 0xb9?, 0x7ffb3a4a5d06?)\n\t/usr/local/go/src/runtime/proc.go:460 +0xce fp=0xc000d9ba90 sp=0xc000d9ba70 pc=0x2ace28e\nruntime.goparkunlock(...)\n\t/usr/local/go/src/runtime/proc.go:466\nruntime.semacquire1(0xc0014c56d0, 0x0, 0x1, 0x0, 0x19)\n\t/usr/local/go/src/runtime/sema.go:192 +0x21d fp=0xc000d9baf8 sp=0xc000d9ba90 pc=0x2aaa3dd\nsync.runtime_SemacquireWaitGroup(0xc001004c00?, 0xc0?)\n\t/usr/local/go/src/runtime/sema.go:114 +0x2e fp=0xc000d9bb30 sp=0xc000d9baf8 pc=0x2acf9ae\nsync.(*WaitGroup).Wait(0xc0014c56c8)\n\t/usr/local/go/src/sync/waitgroup.go:206 +0x85 fp=0xc000d9bb58 sp=0xc000d9bb30 pc=0x2af2e05\ngolang.org/x/sync/errgroup.(*Group).Wait(0xc0014c56c0)\n\t/go/pkg/mod/golang.org/x/sync@v0.19.0/errgroup/errgroup.go:56 +0x1e fp=0xc000d9bb78 sp=0xc000d9bb58 pc=0x376d67e\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters({0x7ffb3818b460, 0xc0016fe180}, 0xc001004540, {0xc000a86dc0, 0x1c}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:319 +0x8ef fp=0xc000d9be20 sp=0xc000d9bb78 pc=0x4eb296f\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).cleanupResourceGroup(0xc0008a4d80, {0x7ffb3818b460, 0xc0016fe180}, {0xc000a86dc0, 0x1c}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:433 +0x294 fp=0xc000d9bf28 sp=0xc000d9be20 pc=0x4ec7c74\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.func1({0x7ffb3818b460, 0xc0016fe180})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:294 +0x14d fp=0xc000d9bfc0 sp=0xc000d9bf28 pc=0x4ec67ad\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:303 +0x28 fp=0xc000d9bfe0 sp=0xc000d9bfc0 pc=0x4ec6628\nruntime.goexit({})\n\t/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000d9bfe8 sp=0xc000d9bfe0 pc=0x2ad6921\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 1382\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:287 +0xc5\n\nrax 0xca\nrbx 0x0\nrcx 0x2ad8743\nrdx 0x0\nrdi 0xb361a90\nrsi 0x80\nrbp 0x7ffd9affcdd0\nrsp 0x7ffd9affcd88\nr8 0x0\nr9 0x0\nr10 0x0\nr11 0x286\nr12 0x8b\nr13 0x1\nr14 0xb35f9a0\nr15 0x1\nrip 0x2ad8741\nrflags 0x286\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"40d4c734a96f001caeb26a6e31a5bbc15267d3bcc6fa3f54f67aefee3bdb011b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4369/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510636851302400","occurred_at":"2026-03-16T11:48:11Z","failed_at":"e2e","signature_id":"afc767c6641eaab0d8ca28d5dc29d3f29211ad450637a1e8fd1a484e36af3480","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001784210\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-mjbrxh\" in resourcegroup=\"admin-credential-lifecycle-test-9znq98\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000fec140\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-mjbrxh\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-9znq98\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000fec120\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-mjbrxh\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-9znq98\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000fec100\u003e{\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-mjbrxh\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-9znq98\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001784210\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-mjbrxh\" in resourcegroup=\"admin-credential-lifecycle-test-9znq98\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000fec140\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-mjbrxh\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-9znq98\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000fec120\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-mjbrxh\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-9znq98\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000fec100\u003e{\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-mjbrxh\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-9znq98\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"62a16c38a99ad529b4fc643b336b15dad016c3f259973420768d77934ae2c3f5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4369/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510636851302400","occurred_at":"2026-03-16T11:48:11Z","failed_at":"e2e","signature_id":"015d304aaa875b2ead960d2146f9410bd3e3187c69e1156b1d51f381119137a2","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0014ea120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-b99kmv\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f9ce00\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-b99kmv\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f9cdc0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-b99kmv\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f9cda0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-b99kmv\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0014ea120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-b99kmv\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f9ce00\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-b99kmv\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f9cdc0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-b99kmv\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f9cda0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-b99kmv\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"65f6b2be6624ec80ee4001530ed1603837307baafb696362dfe11dc97ca4337f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4369/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510636851302400","occurred_at":"2026-03-16T11:48:11Z","failed_at":"e2e","signature_id":"ca3798d3a4dd8c163222e0098e5c66febda7d1f3d3c6597f3c1908c6403a16f5","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0016d0348\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-gqdks9\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00130a2c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-gqdks9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00130a2a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-gqdks9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00130a280\u003e{\n msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-gqdks9\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0016d0348\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-gqdks9\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00130a2c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-gqdks9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00130a2a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-gqdks9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00130a280\u003e{\n msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-gqdks9\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"6ef709f12b46102ef4c21700d7c96ac953a2afc2cf0400e22d69b3fad7301165","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4369/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510636851302400","occurred_at":"2026-03-16T11:48:11Z","failed_at":"e2e","signature_id":"1525eeff8308630e02ea6799be29591c425dfeae2899c49aa325f62b886817f3","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d9c270\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-2tztp4\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000d8d4e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-2tztp4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d8d4c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-2tztp4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d8d4a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-2tztp4\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d9c270\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-2tztp4\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000d8d4e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-2tztp4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d8d4c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-2tztp4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d8d4a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-2tztp4\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"842963c55a0eed213bc893ef75a31faa66bc5a6b2c34c6621300a46f887b6693","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4369/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510636851302400","occurred_at":"2026-03-16T11:48:11Z","failed_at":"e2e","signature_id":"9f0a40543ddda796c80d8d33d38ad7518dfd2ff5ba7f9eaf51665951c02b9248","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001170348\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-vfl76k\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0005904a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-vfl76k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000590460\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-vfl76k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000590440\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-vfl76k\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001170348\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-vfl76k\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0005904a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-vfl76k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000590460\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-vfl76k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000590440\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-vfl76k\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"8cc6fe55f82304cce2ff0e5bb097c3430b95e0f93ebba85c0f32d5077c2abcee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4369/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510636851302400","occurred_at":"2026-03-16T11:48:11Z","failed_at":"e2e","signature_id":"47424b37ccab30ff3240f90ef149d00308e7219621d9b5c5b21235b232126b3c","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [:0]: A node timeout occurred\nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"697cf2a8e0066c5a6c26e4bfa087729d291febf4ff440c7e19b4613b871c7bd7","failure_pattern":"fail \u003cip\u003e: A node timeout occurred","search_query":"fail [:0]: A node timeout occurred"}},{"row_id":"9ce893cd05936b4f8b2315894358b836351f4c6bd72b94c6e99b51040055f013","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4369/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510636851302400","occurred_at":"2026-03-16T11:48:11Z","failed_at":"e2e","signature_id":"83ae208e5c05add23065d82ac6035239fe08a659ca32abb3a14146b36cb7dc04","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0007c6ab0\u003e: \n failed to cleanup resource group: failed waiting for resourcegroup=\"e2e-cidr-connectivity-n5jtqq\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0010cec40\u003e{\n msg: \"failed to cleanup resource group: failed waiting for resourcegroup=\\\"e2e-cidr-connectivity-n5jtqq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0010cec20\u003e{\n msg: \"failed waiting for resourcegroup=\\\"e2e-cidr-connectivity-n5jtqq\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0007c6ab0\u003e: \n failed to cleanup resource group: failed waiting for resourcegroup=\"e2e-cidr-connectivity-n5jtqq\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0010cec40\u003e{\n msg: \"failed to cleanup resource group: failed waiting for resourcegroup=\\\"e2e-cidr-connectivity-n5jtqq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0010cec20\u003e{\n msg: \"failed waiting for resourcegroup=\\\"e2e-cidr-connectivity-n5jtqq\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d55c4a29bbbf19a43335bfa2157018ebc82a1c8fe8cbea22f78ef2ca4c570f1a","failure_pattern":"failed to cleanup resource group: failed waiting for resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: failed waiting for resourcegroup=\"autoscaling-cluster-2bvkgc\" to finish deleting: context canceled"}},{"row_id":"c022b233ecfa92904b218831a1fdeb51b4c8de78adfef31dfb0004e73390ddd6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4369/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510636851302400","occurred_at":"2026-03-16T11:48:11Z","failed_at":"e2e","signature_id":"0853d7c96d317a6c67f9d8ead09b38e5c69fad677400bc54bcf1ec9db8d7cc79","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f32270\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-8rvdx2\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00077b180\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-8rvdx2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00077b160\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-8rvdx2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00077b140\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-8rvdx2\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f32270\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-8rvdx2\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00077b180\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-8rvdx2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00077b160\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-8rvdx2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00077b140\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-8rvdx2\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"c37c4d606d8f2af5dcd6880568346614bebeccde335f1f88d0ef9732c2d0835d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4369/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510636851302400","occurred_at":"2026-03-16T11:48:11Z","failed_at":"e2e","signature_id":"972784d6554fda6ea83fb94056bab0ea900856e24b016c1b8a245b8480dc677f","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d7a300\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-xfpb7p\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001075100\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-xfpb7p\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0010750e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-xfpb7p\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0010750c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-xfpb7p\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d7a300\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-xfpb7p\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001075100\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-xfpb7p\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0010750e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-xfpb7p\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0010750c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-xfpb7p\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"dbe2b5483e6f754004490ea1a368a88dac352c164fd45114eca114f388a000e6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4369/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510636851302400","occurred_at":"2026-03-16T11:48:11Z","failed_at":"e2e","signature_id":"9f04af0d406fd36128e1c370e2706c0dd62eb5df59688f1201d6db4ccda7f35d","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010ca258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-9zglhj\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007938e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-9zglhj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007938c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-9zglhj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007938a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-9zglhj\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010ca258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-9zglhj\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007938e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-9zglhj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007938c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-9zglhj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007938a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-9zglhj\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"f5920cc5de022c840c3b539bff498314dd3fa3f59be105e45c823aa8aeda3079","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4369/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510636851302400","occurred_at":"2026-03-16T11:48:11Z","failed_at":"e2e","signature_id":"e5b2eeb7f5cf12c94cf972a80c4e9a6871947fe64261cccc74114b9239a4ec7c","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:137]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e143c0\u003e: \n route host was never found: Get \"https://agnhost-e2e-serving-app-sckjc.apps.aro.u0e2e1n2t9u1a8h.4rck.j1302400.hcp.osadev.cloud\": dial tcp: lookup agnhost-e2e-serving-app-sckjc.apps.aro.u0e2e1n2t9u1a8h.4rck.j1302400.hcp.osadev.cloud on 172.30.0.10:53: no such host\n {\n msg: \"route host was never found: Get \\\"https://agnhost-e2e-serving-app-sckjc.apps.aro.u0e2e1n2t9u1a8h.4rck.j1302400.hcp.osadev.cloud\\\": dial tcp: lookup agnhost-e2e-serving-app-sckjc.apps.aro.u0e2e1n2t9u1a8h.4rck.j1302400.hcp.osadev.cloud on 172.30.0.10:53: no such host\",\n err: \u003c*url.Error | 0xc000d7f8c0\u003e{\n Op: \"Get\",\n URL: \"https://agnhost-e2e-serving-app-sckjc.apps.aro.u0e2e1n2t9u1a8h.4rck.j1302400.hcp.osadev.cloud\",\n Err: \u003c*net.OpError | 0xc000d72550\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xc000a534a0\u003e{\n UnwrapErr: nil,\n Err: \"no such host\",\n Name: \"agnhost-e2e-serving-app-sckjc.apps.aro.u0e2e1n2t9u1a8h.4rck.j1302400.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: false,\n IsTemporary: false,\n IsNotFound: true,\n },\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:137]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e143c0\u003e: \n route host was never found: Get \"https://agnhost-e2e-serving-app-sckjc.apps.aro.u0e2e1n2t9u1a8h.4rck.j1302400.hcp.osadev.cloud\": dial tcp: lookup agnhost-e2e-serving-app-sckjc.apps.aro.u0e2e1n2t9u1a8h.4rck.j1302400.hcp.osadev.cloud on 172.30.0.10:53: no such host\n {\n msg: \"route host was never found: Get \\\"https://agnhost-e2e-serving-app-sckjc.apps.aro.u0e2e1n2t9u1a8h.4rck.j1302400.hcp.osadev.cloud\\\": dial tcp: lookup agnhost-e2e-serving-app-sckjc.apps.aro.u0e2e1n2t9u1a8h.4rck.j1302400.hcp.osadev.cloud on 172.30.0.10:53: no such host\",\n err: \u003c*url.Error | 0xc000d7f8c0\u003e{\n Op: \"Get\",\n URL: \"https://agnhost-e2e-serving-app-sckjc.apps.aro.u0e2e1n2t9u1a8h.4rck.j1302400.hcp.osadev.cloud\",\n Err: \u003c*net.OpError | 0xc000d72550\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xc000a534a0\u003e{\n UnwrapErr: nil,\n Err: \"no such host\",\n Name: \"agnhost-e2e-serving-app-sckjc.apps.aro.u0e2e1n2t9u1a8h.4rck.j1302400.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: false,\n IsTemporary: false,\n IsNotFound: true,\n },\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fd93f6786bdbe9aeb08610dcc4ec01a349d1a8e658826e9448f8193966d5a67f","failure_pattern":"route host was never found: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"route host was never found: Get \"https://agnhost-e2e-serving-app-9f7x5.apps.aro.l6q3l5t4y9r4i6k.15br.j8542976.hcp.osadev.cloud\": dial tcp: lookup agnhost-e2e-serving-app-9f7x5.apps.aro.l6q3l5t4y9r4i6k.15br.j8542976.hcp.osadev.cloud on 172.30.0.10:53: no such host"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","697cf2a8e0066c5a6c26e4bfa087729d291febf4ff440c7e19b4613b871c7bd7","9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","d55c4a29bbbf19a43335bfa2157018ebc82a1c8fe8cbea22f78ef2ca4c570f1a","fd93f6786bdbe9aeb08610dcc4ec01a349d1a8e658826e9448f8193966d5a67f"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4370/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510537173667840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4370,"pr_state":"open","pr_sha":"2a2dae464713d6ab2c6a447312f5886a4363e8cd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:47:47Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ef26523537af9f62548557fd19cf14709ec33c3ee70ab607eb75942e4b284105","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4370/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510537173667840","occurred_at":"2026-03-16T11:47:47Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4375/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510189960794112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4375,"pr_state":"closed","pr_sha":"5f83abef619af4278148a16c27c699f25a00c7a4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:46:23Z"},"failed_at":["e2e"],"failed_test_count":17,"occurrences":[{"row_id":"08c35e61db371de32072311dd1b15dba1608db39fb6fbe36567eb95859146812","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4375/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510189960794112","occurred_at":"2026-03-16T11:46:23Z","failed_at":"e2e","signature_id":"b47d438c66f581e0844acadca0fea556f9c3ccff83afc0ef8376d2debfb31c0c","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0000f8ba0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-4ch82w\" to finish creating: GET https://rp.j0794112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26153cef-41b7-4eae-9a57-cc0a81a7c19d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26153cef-41b7-4eae-9a57-cc0a81a7c19d\",\n \"name\": \"26153cef-41b7-4eae-9a57-cc0a81a7c19d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:29:53.139498332Z\",\n \"endTime\": \"2026-03-16T14:31:10.125431656Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-135/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-135/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: 429\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"429\\\",\\n \\\"message\\\": \\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 2 seconds.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-4ch82w\\\" to finish creating: GET https://rp.j0794112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26153cef-41b7-4eae-9a57-cc0a81a7c19d\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26153cef-41b7-4eae-9a57-cc0a81a7c19d\\\",\\n \\\"name\\\": \\\"26153cef-41b7-4eae-9a57-cc0a81a7c19d\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:29:53.139498332Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:31:10.125431656Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-135/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-135/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: 429\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"429\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 2 seconds.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0000f8b60\u003e{\n msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-4ch82w\\\" to finish creating: GET https://rp.j0794112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26153cef-41b7-4eae-9a57-cc0a81a7c19d\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26153cef-41b7-4eae-9a57-cc0a81a7c19d\\\",\\n \\\"name\\\": \\\"26153cef-41b7-4eae-9a57-cc0a81a7c19d\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:29:53.139498332Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:31:10.125431656Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-135/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-135/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: 429\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"429\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Request is temporarily throttled because...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0000f8ba0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-4ch82w\" to finish creating: GET https://rp.j0794112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26153cef-41b7-4eae-9a57-cc0a81a7c19d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26153cef-41b7-4eae-9a57-cc0a81a7c19d\",\n \"name\": \"26153cef-41b7-4eae-9a57-cc0a81a7c19d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:29:53.139498332Z\",\n \"endTime\": \"2026-03-16T14:31:10.125431656Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-135/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-135/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: 429\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"429\\\",\\n \\\"message\\\": \\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 2 seconds.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-4ch82w\\\" to finish creating: GET https://rp.j0794112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26153cef-41b7-4eae-9a57-cc0a81a7c19d\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26153cef-41b7-4eae-9a57-cc0a81a7c19d\\\",\\n \\\"name\\\": \\\"26153cef-41b7-4eae-9a57-cc0a81a7c19d\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:29:53.139498332Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:31:10.125431656Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-135/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-135/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: 429\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"429\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 2 seconds.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0000f8b60\u003e{\n msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-4ch82w\\\" to finish creating: GET https://rp.j0794112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26153cef-41b7-4eae-9a57-cc0a81a7c19d\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26153cef-41b7-4eae-9a57-cc0a81a7c19d\\\",\\n \\\"name\\\": \\\"26153cef-41b7-4eae-9a57-cc0a81a7c19d\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:29:53.139498332Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:31:10.125431656Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-135/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-135/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: 429\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"429\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Request is temporarily throttled because...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"30fa2fa857c42c0b459f74a2bbc1050bfe224ec724cca8fb5c955cd9abb6b04a","failure_pattern":"ERROR CODE: InternalServerError; detail code 429; detail message Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 2 seconds.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"09b2c1e37c2b4a2905093c2bc604b00bf1f90140f2f4870b29f364eff5feae4e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4375/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510189960794112","occurred_at":"2026-03-16T11:46:23Z","failed_at":"e2e","signature_id":"2aa51fbfb05f1e1f647f28c4643a812254aca61fcca058641cb5d556b3240f0f","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000fb4258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-7j9bkr\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000805a80\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-7j9bkr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000805a60\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-7j9bkr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000805a40\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-7j9bkr\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000fb4258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-7j9bkr\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000805a80\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-7j9bkr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000805a60\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-7j9bkr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000805a40\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-7j9bkr\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"0dfa65b86b8e25f4f158c7a52f89da1a4c0802b817016d80def653d92810d53e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4375/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510189960794112","occurred_at":"2026-03-16T11:46:23Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:152]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:152]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"140dacd120748b63f25838dd5375bdbb5e51b6522565ca889c34daf3fef1b444","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4375/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510189960794112","occurred_at":"2026-03-16T11:46:23Z","failed_at":"e2e","signature_id":"1699e2a37d1c89913eac573cd39c8b2592368fe8b016ddeddfcc4e580f7e7e3b","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00100f0b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-h5jp46\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007937e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-h5jp46\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007937c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-h5jp46\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007937a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-h5jp46\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00100f0b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-h5jp46\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007937e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-h5jp46\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007937c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-h5jp46\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007937a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-h5jp46\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"1f3881b25145ee0fa847684ee167ac3ba31ed140c9e92b8ee55d34ec0561cb32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4375/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510189960794112","occurred_at":"2026-03-16T11:46:23Z","failed_at":"e2e","signature_id":"17f14f22b4ba1f5a101415084162a12ef93a29f9789831a584c4aea68645c1f7","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00111a3f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listnhkznp\" in resourcegroup=\"ea-list-r74njm\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0008a8600\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listnhkznp\\\" in resourcegroup=\\\"ea-list-r74njm\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008a85e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listnhkznp\\\" in resourcegroup=\\\"ea-list-r74njm\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008a85c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listnhkznp\\\" in resourcegroup=\\\"ea-list-r74njm\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00111a3f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listnhkznp\" in resourcegroup=\"ea-list-r74njm\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0008a8600\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listnhkznp\\\" in resourcegroup=\\\"ea-list-r74njm\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008a85e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listnhkznp\\\" in resourcegroup=\\\"ea-list-r74njm\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008a85c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listnhkznp\\\" in resourcegroup=\\\"ea-list-r74njm\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"337adce38cc292728ee7767e0bd5916496da8b5792f56aec3e3f7d007fc1f5d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4375/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510189960794112","occurred_at":"2026-03-16T11:46:23Z","failed_at":"e2e","signature_id":"f3dd536349ddf19b905261a0eecf957df0ec9771fc69c8142ef145ddd534346b","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0007a4da0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-c4jlq6: failed waiting for cluster=\"cluster-zstream-4-20-c4jlq6\" in resourcegroup=\"rg-zstream-upgrade-4-20-zx9bm6\" to finish creating: GET https://rp.j0794112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b34b3935-cea1-4830-80d1-79ff5a763ec5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b34b3935-cea1-4830-80d1-79ff5a763ec5\",\n \"name\": \"b34b3935-cea1-4830-80d1-79ff5a763ec5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:35:16.621213572Z\",\n \"endTime\": \"2026-03-16T14:36:17.621952691Z\",\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"Cluster provisioning failed due to multiple errors\",\n \"details\": [\n {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-129/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-129/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n },\n {\n \"code\": \"InternalServerError\",\n \"message\": \"Failed to get virtual network 'customer-vnet-zstream-c4jlq6' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-zx9bm6/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-c4jlq6\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-20-c4jlq6: failed waiting for cluster=\\\"cluster-zstream-4-20-c4jlq6\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-zx9bm6\\\" to finish creating: GET https://rp.j0794112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b34b3935-cea1-4830-80d1-79ff5a763ec5\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b34b3935-cea1-4830-80d1-79ff5a763ec5\\\",\\n \\\"name\\\": \\\"b34b3935-cea1-4830-80d1-79ff5a763ec5\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:35:16.621213572Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:36:17.621952691Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-129/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-129/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n },\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-zstream-c4jlq6' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-zx9bm6/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-c4jlq6\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0007a4d80\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-c4jlq6\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-zx9bm6\\\" to finish creating: GET https://rp.j0794112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b34b3935-cea1-4830...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0007a4da0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-c4jlq6: failed waiting for cluster=\"cluster-zstream-4-20-c4jlq6\" in resourcegroup=\"rg-zstream-upgrade-4-20-zx9bm6\" to finish creating: GET https://rp.j0794112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b34b3935-cea1-4830-80d1-79ff5a763ec5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b34b3935-cea1-4830-80d1-79ff5a763ec5\",\n \"name\": \"b34b3935-cea1-4830-80d1-79ff5a763ec5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:35:16.621213572Z\",\n \"endTime\": \"2026-03-16T14:36:17.621952691Z\",\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"Cluster provisioning failed due to multiple errors\",\n \"details\": [\n {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-129/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-129/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n },\n {\n \"code\": \"InternalServerError\",\n \"message\": \"Failed to get virtual network 'customer-vnet-zstream-c4jlq6' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-zx9bm6/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-c4jlq6\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-20-c4jlq6: failed waiting for cluster=\\\"cluster-zstream-4-20-c4jlq6\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-zx9bm6\\\" to finish creating: GET https://rp.j0794112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b34b3935-cea1-4830-80d1-79ff5a763ec5\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b34b3935-cea1-4830-80d1-79ff5a763ec5\\\",\\n \\\"name\\\": \\\"b34b3935-cea1-4830-80d1-79ff5a763ec5\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:35:16.621213572Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:36:17.621952691Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-129/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-129/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n },\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-zstream-c4jlq6' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-zx9bm6/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-c4jlq6\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0007a4d80\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-c4jlq6\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-zx9bm6\\\" to finish creating: GET https://rp.j0794112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b34b3935-cea1-4830...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0aa9ed5d94e7167a12aa30282eff19539fc3fc473772cb96b223d420df66775f","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"4282ba056b77190b11310544716595ab20bf5e4da41bddb8ba8c4c03460364c7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4375/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510189960794112","occurred_at":"2026-03-16T11:46:23Z","failed_at":"e2e","signature_id":"c3fa31d3876441cb111ea0ae21ab00b5087ca5615c2d1e5e097a324230cad50a","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.007s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00099d390\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.007s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00099d390\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}},{"row_id":"454db490a25585fd2fcecd5880ea68ff33bff1211657f02bf2a80325c50ff342","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4375/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510189960794112","occurred_at":"2026-03-16T11:46:23Z","failed_at":"e2e","signature_id":"1820f8d1a1eb69e026e6ba27a5978cbc331e0e84249764492674f61f160c9c29","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:73]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000ba81e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-update-nodes-hcp-cluster-pxjp76\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-pxjp76/operationStatuses/08584279348946056859\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-s8kh7j/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-136/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-np-update-nodes-hcp-cluster-pxjp76\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-pxjp76/operationStatuses/08584279348946056859\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-s8kh7j/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:73]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000ba81e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-update-nodes-hcp-cluster-pxjp76\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-pxjp76/operationStatuses/08584279348946056859\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-s8kh7j/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-136/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-np-update-nodes-hcp-cluster-pxjp76\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-pxjp76/operationStatuses/08584279348946056859\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-s8kh7j/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"4a8bd9ac4fab0dcf487151fd5a866273bfa030a65192897dfe5be6791ae69e08","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4375/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510189960794112","occurred_at":"2026-03-16T11:46:23Z","failed_at":"e2e","signature_id":"713c94bf77d25f17d9fcbef1514d199914853a4883ff2e8a4957c4cd319237cd","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000fa82e8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-m5vdx7\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0013b66c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-m5vdx7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0013b66a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-m5vdx7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001129260\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-m5vdx7\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000fa82e8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-m5vdx7\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0013b66c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-m5vdx7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0013b66a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-m5vdx7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001129260\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-m5vdx7\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"779f08cd343c19852c90ba21ccd6efd2ac9404bcf17a05c4a950724233b8ec9f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4375/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510189960794112","occurred_at":"2026-03-16T11:46:23Z","failed_at":"e2e","signature_id":"eae59c5f5cbf03acf3b8df69d8703e1709be5911d60194dc1a78367a6c5e7290","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0010d2140\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-92x7pp\" to finish creating: GET https://rp.j0794112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb40865a-f041-407f-b523-d62555f0b51e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb40865a-f041-407f-b523-d62555f0b51e\",\n \"name\": \"cb40865a-f041-407f-b523-d62555f0b51e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:35:15.551650129Z\",\n \"endTime\": \"2026-03-16T14:36:17.644274728Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-136/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-136/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-92x7pp\\\" to finish creating: GET https://rp.j0794112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb40865a-f041-407f-b523-d62555f0b51e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb40865a-f041-407f-b523-d62555f0b51e\\\",\\n \\\"name\\\": \\\"cb40865a-f041-407f-b523-d62555f0b51e\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:35:15.551650129Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:36:17.644274728Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-136/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-136/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0010d2120\u003e{\n msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-92x7pp\\\" to finish creating: GET https://rp.j0794112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb40865a-f041-407f-b523-d62555f0b51e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb40865a-f041-407f-b523-d62555f0b51e\\\",\\n \\\"name\\\": \\\"cb40865a-f041-407f-b523-d62555f0b51e\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:35:15.551650129Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:36:17.644274728Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-136/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-136/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n----------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0010d2140\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-92x7pp\" to finish creating: GET https://rp.j0794112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb40865a-f041-407f-b523-d62555f0b51e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb40865a-f041-407f-b523-d62555f0b51e\",\n \"name\": \"cb40865a-f041-407f-b523-d62555f0b51e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:35:15.551650129Z\",\n \"endTime\": \"2026-03-16T14:36:17.644274728Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-136/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-136/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-92x7pp\\\" to finish creating: GET https://rp.j0794112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb40865a-f041-407f-b523-d62555f0b51e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb40865a-f041-407f-b523-d62555f0b51e\\\",\\n \\\"name\\\": \\\"cb40865a-f041-407f-b523-d62555f0b51e\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:35:15.551650129Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:36:17.644274728Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-136/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-136/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0010d2120\u003e{\n msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-92x7pp\\\" to finish creating: GET https://rp.j0794112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb40865a-f041-407f-b523-d62555f0b51e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb40865a-f041-407f-b523-d62555f0b51e\\\",\\n \\\"name\\\": \\\"cb40865a-f041-407f-b523-d62555f0b51e\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:35:15.551650129Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:36:17.644274728Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-136/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-136/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n----------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9bc5097090b62957eac4fdf8658d70dadcdf169bb60bb63e33de1195f64b6be9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4375/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510189960794112","occurred_at":"2026-03-16T11:46:23Z","failed_at":"e2e","signature_id":"48bdf7fc7276331232fab73b757d589947b3408cd6134a4819de310e99aee1c1","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d56540\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-nw2rtr\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00080d160\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-nw2rtr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00080d140\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-nw2rtr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00080d120\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-nw2rtr\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d56540\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-nw2rtr\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00080d160\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-nw2rtr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00080d140\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-nw2rtr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00080d120\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-nw2rtr\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"9d5953cda1b8b932f0880f6f3ae003b8331dd4455c3add09cde002f1147a76d7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4375/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510189960794112","occurred_at":"2026-03-16T11:46:23Z","failed_at":"e2e","signature_id":"62dc98fdcc5bc54cd57b99275a32ba09bf22fba8afa692d3e7e76649e12599d1","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00157a018\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-n29hzc\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000021e20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-n29hzc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000021e00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-n29hzc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000021de0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-n29hzc\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00157a018\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-n29hzc\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000021e20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-n29hzc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000021e00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-n29hzc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000021de0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-n29hzc\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"a3229cda77f505a3666d5cdeafb4e9364a29a799584cc1aba4db8632a60a98cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4375/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510189960794112","occurred_at":"2026-03-16T11:46:23Z","failed_at":"e2e","signature_id":"f86136a1533cfdcda88d0d2350258337a88b11732479b41c610c146941d9c67f","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f301b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-q9z6gj\" in resourcegroup=\"rg-zstream-upgrade-4-21-thqc42\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0008074c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-q9z6gj\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-thqc42\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008074a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-q9z6gj\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-thqc42\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000807460\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-q9z6gj\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-thqc42\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f301b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-q9z6gj\" in resourcegroup=\"rg-zstream-upgrade-4-21-thqc42\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0008074c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-q9z6gj\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-thqc42\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008074a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-q9z6gj\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-thqc42\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000807460\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-q9z6gj\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-thqc42\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"b44782ff1c79080af4aa7d8c34144ed24e7396d40a13256ff4bdd4b360765180","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4375/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510189960794112","occurred_at":"2026-03-16T11:46:23Z","failed_at":"e2e","signature_id":"3f4a975ef9231e42a9f5734cea4548351c9b5a092b2b22a5236b4f8dee58c545","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0011c81c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-9wjtwl\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000d25040\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-9wjtwl\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d25020\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-9wjtwl\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d25000\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-9wjtwl\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0011c81c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-9wjtwl\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000d25040\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-9wjtwl\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d25020\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-9wjtwl\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d25000\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-9wjtwl\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"dd50ca41b87a9168804649cd6632c1593a73fb99f30476b2f5cf8577ba0eaaf9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4375/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510189960794112","occurred_at":"2026-03-16T11:46:23Z","failed_at":"e2e","signature_id":"0a8a8080eca5c3bb479bd0910a48f9f928b0a5defe81a4b6153fc2bba18c6bed","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001194108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-6cnm8w\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000df6e60\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-6cnm8w\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000df6e20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-6cnm8w\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000df6e00\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-6cnm8w\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001194108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-6cnm8w\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000df6e60\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-6cnm8w\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000df6e20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-6cnm8w\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000df6e00\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-6cnm8w\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"e5efb42c3dbf0c805f508f62381f1ac924fc27e975f68888e32ad5ab120ba223","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4375/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510189960794112","occurred_at":"2026-03-16T11:46:23Z","failed_at":"e2e","signature_id":"a6d374ef51b0de4cd11bb3780205979546deb2af6d796e02be103e9621de8c7d","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00112a180\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-x2g6p2\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0013770c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-x2g6p2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0013770a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-x2g6p2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001377040\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-x2g6p2\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00112a180\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-x2g6p2\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0013770c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-x2g6p2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0013770a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-x2g6p2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001377040\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-x2g6p2\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"ea4d4dbaa41091de5377fd2c17b81683d4033e1e1fd0f588a07926429cda4c33","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4375/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510189960794112","occurred_at":"2026-03-16T11:46:23Z","failed_at":"e2e","signature_id":"e93843e37f96c9fb6a74dcd3e73d576380324436a10d86d5fbff62f57843bdcb","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001026318\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-7x952s\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000779c60\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-7x952s\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000779c20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-7x952s\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000779c00\u003e{\n msg: \"failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-7x952s\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001026318\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-7x952s\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000779c60\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-7x952s\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000779c20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-7x952s\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000779c00\u003e{\n msg: \"failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-7x952s\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":17,"failure_pattern_ids":["0aa9ed5d94e7167a12aa30282eff19539fc3fc473772cb96b223d420df66775f","30fa2fa857c42c0b459f74a2bbc1050bfe224ec724cca8fb5c955cd9abb6b04a","503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e"],"matched_occurrences":17,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4385/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510083974926336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4385,"pr_state":"closed","pr_sha":"07e421bae19659fd95e6d16899bed8ad2fda3863","final_merged_sha":"07e421bae19659fd95e6d16899bed8ad2fda3863","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T11:45:59Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ecf64c5d9bfc342f39b45a047d71e89bd251277395d0f2782e0a0c26a614dd63","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4385/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033510083974926336","occurred_at":"2026-03-16T11:45:59Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4391/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509882786746368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4391,"pr_state":"open","pr_sha":"dacf87647418e205a2b119537d656f914a585006","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:45:11Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"16425f8076a991175021dfc155066e86076c50f8357a5b5afb222638b00af985","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4391/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509882786746368","occurred_at":"2026-03-16T11:45:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509681468542976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4393,"pr_state":"open","pr_sha":"dcec1a0f0e9fc5822919f091837faed741634449","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:44:23Z"},"failed_at":["e2e"],"failed_test_count":11,"occurrences":[{"row_id":"1406aeb8ba0e6082bfa12a603e9375066e1d9fa970c8ac3618ce1d573659d0b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509681468542976","occurred_at":"2026-03-16T11:44:23Z","failed_at":"e2e","signature_id":"7f9db0b819dc6a9eacb68f820ef742e91485b9581677da44a8192047f32f9208","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:89]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001582140\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-fwc64f\" to finish creating: GET https://rp.j8542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/647c9287-bdeb-4768-8590-42e0cce48fea\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/647c9287-bdeb-4768-8590-42e0cce48fea\",\n \"name\": \"647c9287-bdeb-4768-8590-42e0cce48fea\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:39:38.774216229Z\",\n \"endTime\": \"2026-03-16T12:41:22.078116405Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-fwc64f/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fwc64f\\\" to finish creating: GET https://rp.j8542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/647c9287-bdeb-4768-8590-42e0cce48fea\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/647c9287-bdeb-4768-8590-42e0cce48fea\\\",\\n \\\"name\\\": \\\"647c9287-bdeb-4768-8590-42e0cce48fea\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:39:38.774216229Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:41:22.078116405Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-fwc64f/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001582120\u003e{\n msg: \"failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fwc64f\\\" to finish creating: GET https://rp.j8542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/647c9287-bdeb-4768-8590-42e0cce48fea\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/647c9287-bdeb-4768-8590-42e0cce48fea\\\",\\n \\\"name\\\": \\\"647c9287-bdeb-4768-8590-42e0cce48fea\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:39:38.774216229Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:41:22.078116405Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-fwc64f/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:89]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001582140\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-fwc64f\" to finish creating: GET https://rp.j8542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/647c9287-bdeb-4768-8590-42e0cce48fea\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/647c9287-bdeb-4768-8590-42e0cce48fea\",\n \"name\": \"647c9287-bdeb-4768-8590-42e0cce48fea\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:39:38.774216229Z\",\n \"endTime\": \"2026-03-16T12:41:22.078116405Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-fwc64f/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fwc64f\\\" to finish creating: GET https://rp.j8542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/647c9287-bdeb-4768-8590-42e0cce48fea\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/647c9287-bdeb-4768-8590-42e0cce48fea\\\",\\n \\\"name\\\": \\\"647c9287-bdeb-4768-8590-42e0cce48fea\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:39:38.774216229Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:41:22.078116405Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-fwc64f/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001582120\u003e{\n msg: \"failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fwc64f\\\" to finish creating: GET https://rp.j8542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/647c9287-bdeb-4768-8590-42e0cce48fea\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/647c9287-bdeb-4768-8590-42e0cce48fea\\\",\\n \\\"name\\\": \\\"647c9287-bdeb-4768-8590-42e0cce48fea\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:39:38.774216229Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:41:22.078116405Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-fwc64f/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"23f0c85f64efadc8e4f26a55dd679c97a476a471fd93049af5ed92606f59f554","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509681468542976","occurred_at":"2026-03-16T11:44:23Z","failed_at":"e2e","signature_id":"3d2f4e2bd2f93d9f0c49ed8fd723e1bb4b113fd97a6080ade61da9af35b56ead","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0017f2150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-4cvsn9\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007addc0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-4cvsn9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007adda0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-4cvsn9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007add80\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-4cvsn9\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0017f2150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-4cvsn9\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007addc0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-4cvsn9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007adda0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-4cvsn9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007add80\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-4cvsn9\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"3878d9f73764184aaa8d9c3b3405b1932f29b25d5a9ef90a5431449493f5c6bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509681468542976","occurred_at":"2026-03-16T11:44:23Z","failed_at":"e2e","signature_id":"ba979ee06cd20db5698d3409155a77af5a11d64f5f4b51e5dcef7a45857eb019","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e4b8c0\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-vf4nkn\" to finish creating: GET https://rp.j8542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/652685d3-2aea-4847-ac21-8f15ad30cedf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/652685d3-2aea-4847-ac21-8f15ad30cedf\",\n \"name\": \"652685d3-2aea-4847-ac21-8f15ad30cedf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:31:53.877846503Z\",\n \"endTime\": \"2026-03-16T12:32:42.631963218Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-104/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-104/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: 429\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"429\\\",\\n \\\"message\\\": \\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 3 seconds.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-vf4nkn\\\" to finish creating: GET https://rp.j8542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/652685d3-2aea-4847-ac21-8f15ad30cedf\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/652685d3-2aea-4847-ac21-8f15ad30cedf\\\",\\n \\\"name\\\": \\\"652685d3-2aea-4847-ac21-8f15ad30cedf\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:31:53.877846503Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:32:42.631963218Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-104/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-104/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: 429\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"429\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 3 seconds.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e4b8a0\u003e{\n msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-vf4nkn\\\" to finish creating: GET https://rp.j8542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/652685d3-2aea-4847-ac21-8f15ad30cedf\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/652685d3-2aea-4847-ac21-8f15ad30cedf\\\",\\n \\\"name\\\": \\\"652685d3-2aea-4847-ac21-8f15ad30cedf\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:31:53.877846503Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:32:42.631963218Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-104/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-104/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: 429\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"429\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e4b8c0\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-vf4nkn\" to finish creating: GET https://rp.j8542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/652685d3-2aea-4847-ac21-8f15ad30cedf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/652685d3-2aea-4847-ac21-8f15ad30cedf\",\n \"name\": \"652685d3-2aea-4847-ac21-8f15ad30cedf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:31:53.877846503Z\",\n \"endTime\": \"2026-03-16T12:32:42.631963218Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-104/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-104/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: 429\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"429\\\",\\n \\\"message\\\": \\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 3 seconds.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-vf4nkn\\\" to finish creating: GET https://rp.j8542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/652685d3-2aea-4847-ac21-8f15ad30cedf\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/652685d3-2aea-4847-ac21-8f15ad30cedf\\\",\\n \\\"name\\\": \\\"652685d3-2aea-4847-ac21-8f15ad30cedf\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:31:53.877846503Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:32:42.631963218Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-104/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-104/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: 429\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"429\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 3 seconds.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e4b8a0\u003e{\n msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-vf4nkn\\\" to finish creating: GET https://rp.j8542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/652685d3-2aea-4847-ac21-8f15ad30cedf\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/652685d3-2aea-4847-ac21-8f15ad30cedf\\\",\\n \\\"name\\\": \\\"652685d3-2aea-4847-ac21-8f15ad30cedf\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:31:53.877846503Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:32:42.631963218Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-104/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-104/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: 429\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"429\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3f672ce590a3e36623ba27892eef00b6b866047042ba8ed4eaf2c503e528aee8","failure_pattern":"ERROR CODE: InternalServerError; detail code 429; detail message Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 3 seconds.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3f5d72bf0d5569f01d8e2d3c07047e6dc9957955f76435a1e529bed8280c8258","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509681468542976","occurred_at":"2026-03-16T11:44:23Z","failed_at":"e2e","signature_id":"46b42ee0a051b7f6bcf9ddbb6f846f5ff0e7b589cebcf69b0cbd7971a32388da","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000552bc0\u003e: \n route host was never found: Get \"https://agnhost-e2e-serving-app-9f7x5.apps.aro.l6q3l5t4y9r4i6k.15br.j8542976.hcp.osadev.cloud\": dial tcp: lookup agnhost-e2e-serving-app-9f7x5.apps.aro.l6q3l5t4y9r4i6k.15br.j8542976.hcp.osadev.cloud on 172.30.0.10:53: no such host\n {\n msg: \"route host was never found: Get \\\"https://agnhost-e2e-serving-app-9f7x5.apps.aro.l6q3l5t4y9r4i6k.15br.j8542976.hcp.osadev.cloud\\\": dial tcp: lookup agnhost-e2e-serving-app-9f7x5.apps.aro.l6q3l5t4y9r4i6k.15br.j8542976.hcp.osadev.cloud on 172.30.0.10:53: no such host\",\n err: \u003c*url.Error | 0xc000ea2d80\u003e{\n Op: \"Get\",\n URL: \"https://agnhost-e2e-serving-app-9f7x5.apps.aro.l6q3l5t4y9r4i6k.15br.j8542976.hcp.osadev.cloud\",\n Err: \u003c*net.OpError | 0xc000d588c0\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xc000ea51d0\u003e{\n UnwrapErr: nil,\n Err: \"no such host\",\n Name: \"agnhost-e2e-serving-app-9f7x5.apps.aro.l6q3l5t4y9r4i6k.15br.j8542976.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: false,\n IsTemporary: false,\n IsNotFound: true,\n },\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000552bc0\u003e: \n route host was never found: Get \"https://agnhost-e2e-serving-app-9f7x5.apps.aro.l6q3l5t4y9r4i6k.15br.j8542976.hcp.osadev.cloud\": dial tcp: lookup agnhost-e2e-serving-app-9f7x5.apps.aro.l6q3l5t4y9r4i6k.15br.j8542976.hcp.osadev.cloud on 172.30.0.10:53: no such host\n {\n msg: \"route host was never found: Get \\\"https://agnhost-e2e-serving-app-9f7x5.apps.aro.l6q3l5t4y9r4i6k.15br.j8542976.hcp.osadev.cloud\\\": dial tcp: lookup agnhost-e2e-serving-app-9f7x5.apps.aro.l6q3l5t4y9r4i6k.15br.j8542976.hcp.osadev.cloud on 172.30.0.10:53: no such host\",\n err: \u003c*url.Error | 0xc000ea2d80\u003e{\n Op: \"Get\",\n URL: \"https://agnhost-e2e-serving-app-9f7x5.apps.aro.l6q3l5t4y9r4i6k.15br.j8542976.hcp.osadev.cloud\",\n Err: \u003c*net.OpError | 0xc000d588c0\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xc000ea51d0\u003e{\n UnwrapErr: nil,\n Err: \"no such host\",\n Name: \"agnhost-e2e-serving-app-9f7x5.apps.aro.l6q3l5t4y9r4i6k.15br.j8542976.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: false,\n IsTemporary: false,\n IsNotFound: true,\n },\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fd93f6786bdbe9aeb08610dcc4ec01a349d1a8e658826e9448f8193966d5a67f","failure_pattern":"route host was never found: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"route host was never found: Get \"https://agnhost-e2e-serving-app-9f7x5.apps.aro.l6q3l5t4y9r4i6k.15br.j8542976.hcp.osadev.cloud\": dial tcp: lookup agnhost-e2e-serving-app-9f7x5.apps.aro.l6q3l5t4y9r4i6k.15br.j8542976.hcp.osadev.cloud on 172.30.0.10:53: no such host"}},{"row_id":"5b179d347977a6f1314d2a6ed3ec9568fb4719a80bd0da67226a5c81a8e07ad1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509681468542976","occurred_at":"2026-03-16T11:44:23Z","failed_at":"e2e","signature_id":"a2156cea2aea5205ac9b0dc5125701fd56be98a13a6bdea704513fdc01abe790","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000b2e0d8\u003e: \n failed to cleanup resource group: failed waiting for resourcegroup=\"oidc-wi-rwm7g2\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0008a90e0\u003e{\n msg: \"failed to cleanup resource group: failed waiting for resourcegroup=\\\"oidc-wi-rwm7g2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008a90c0\u003e{\n msg: \"failed waiting for resourcegroup=\\\"oidc-wi-rwm7g2\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000b2e0d8\u003e: \n failed to cleanup resource group: failed waiting for resourcegroup=\"oidc-wi-rwm7g2\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0008a90e0\u003e{\n msg: \"failed to cleanup resource group: failed waiting for resourcegroup=\\\"oidc-wi-rwm7g2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008a90c0\u003e{\n msg: \"failed waiting for resourcegroup=\\\"oidc-wi-rwm7g2\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d55c4a29bbbf19a43335bfa2157018ebc82a1c8fe8cbea22f78ef2ca4c570f1a","failure_pattern":"failed to cleanup resource group: failed waiting for resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: failed waiting for resourcegroup=\"autoscaling-cluster-2bvkgc\" to finish deleting: context canceled"}},{"row_id":"a2db05d5c2d1c6ef49c7c66292d36c2b16728498b78294bce4cb8dc963766605","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509681468542976","occurred_at":"2026-03-16T11:44:23Z","failed_at":"e2e","signature_id":"b287e0abdc8fe663d28e9a7c6e79b56e7e3fc014c4802bca2bea12391c4e2f86","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000dfe3a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-5f4chs\" in resourcegroup=\"rg-zstream-upgrade-4-21-mpmgpd\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0008a20e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-5f4chs\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-mpmgpd\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008a20c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-5f4chs\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-mpmgpd\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008a20a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-5f4chs\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-mpmgpd\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000dfe3a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-5f4chs\" in resourcegroup=\"rg-zstream-upgrade-4-21-mpmgpd\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0008a20e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-5f4chs\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-mpmgpd\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008a20c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-5f4chs\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-mpmgpd\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008a20a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-5f4chs\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-mpmgpd\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"b86aad36763bfff08f3cae8989968c1504595ba526c5d3fedcc0235dbf0767d3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509681468542976","occurred_at":"2026-03-16T11:44:23Z","failed_at":"e2e","signature_id":"ec29e97b4fd7b4e52dc36d4f688b81f28c7d13253b6209e45c5539b2721cd7a2","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e961f8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-t6wj87\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001628360\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-t6wj87\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001628340\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-t6wj87\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001628320\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-t6wj87\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e961f8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-t6wj87\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001628360\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-t6wj87\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001628340\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-t6wj87\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001628320\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-t6wj87\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"c03da53b2b5d4f6ebd470b27d8d0c0a44a6f78730a5051ca8ada15b84a25fcf1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509681468542976","occurred_at":"2026-03-16T11:44:23Z","failed_at":"e2e","signature_id":"701d563cb39424195dd54c5fda25ecef402622e310ba3ff0e3d0dc5038906d76","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0018900f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-qr7jf6\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0009c6dc0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-qr7jf6\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009c6da0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-qr7jf6\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009c6d80\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-qr7jf6\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0018900f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-qr7jf6\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0009c6dc0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-qr7jf6\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009c6da0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-qr7jf6\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009c6d80\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-qr7jf6\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"cb4bd7ed09f5bcc75b7be3583cbd01109e1d1c6f830841ca19fd67feff75b632","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509681468542976","occurred_at":"2026-03-16T11:44:23Z","failed_at":"e2e","signature_id":"b9fc6271535a868b7051ff4c539b67536727375fa9b1a43240a5db7641d2f770","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000c88da0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-bhdxzp\" to finish creating: GET https://rp.j8542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67726f63-18de-4791-a065-5830cb428b43\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67726f63-18de-4791-a065-5830cb428b43\",\n \"name\": \"67726f63-18de-4791-a065-5830cb428b43\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:33:49.291317947Z\",\n \"endTime\": \"2026-03-16T12:35:21.638439699Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-128/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-128/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-bhdxzp\\\" to finish creating: GET https://rp.j8542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67726f63-18de-4791-a065-5830cb428b43\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67726f63-18de-4791-a065-5830cb428b43\\\",\\n \\\"name\\\": \\\"67726f63-18de-4791-a065-5830cb428b43\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:33:49.291317947Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:35:21.638439699Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-128/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-128/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000c88d80\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-bhdxzp\\\" to finish creating: GET https://rp.j8542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67726f63-18de-4791-a065-5830cb428b43\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67726f63-18de-4791-a065-5830cb428b43\\\",\\n \\\"name\\\": \\\"67726f63-18de-4791-a065-5830cb428b43\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:33:49.291317947Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:35:21.638439699Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-128/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-128/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000c88da0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-bhdxzp\" to finish creating: GET https://rp.j8542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67726f63-18de-4791-a065-5830cb428b43\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67726f63-18de-4791-a065-5830cb428b43\",\n \"name\": \"67726f63-18de-4791-a065-5830cb428b43\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:33:49.291317947Z\",\n \"endTime\": \"2026-03-16T12:35:21.638439699Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-128/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-128/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-bhdxzp\\\" to finish creating: GET https://rp.j8542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67726f63-18de-4791-a065-5830cb428b43\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67726f63-18de-4791-a065-5830cb428b43\\\",\\n \\\"name\\\": \\\"67726f63-18de-4791-a065-5830cb428b43\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:33:49.291317947Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:35:21.638439699Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-128/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-128/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000c88d80\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-bhdxzp\\\" to finish creating: GET https://rp.j8542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67726f63-18de-4791-a065-5830cb428b43\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67726f63-18de-4791-a065-5830cb428b43\\\",\\n \\\"name\\\": \\\"67726f63-18de-4791-a065-5830cb428b43\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:33:49.291317947Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:35:21.638439699Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-128/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-128/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"dd365e770a2ac1c1dbb9ba473882944bd7611584ec334ce1bffa5f02981bc0bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509681468542976","occurred_at":"2026-03-16T11:44:23Z","failed_at":"e2e","signature_id":"e25076d26606881d869a46dcbd3399990090e24e6cd74a490e39c9313f67d1f7","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000c70ed0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-19-vvj692\" in resourcegroup=\"rg-zstream-upgrade-4-19-sr4x5d\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001041560\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-19-vvj692\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-sr4x5d\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001041540\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-19-vvj692\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-sr4x5d\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001041520\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-vvj692\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-sr4x5d\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000c70ed0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-19-vvj692\" in resourcegroup=\"rg-zstream-upgrade-4-19-sr4x5d\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001041560\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-19-vvj692\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-sr4x5d\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001041540\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-19-vvj692\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-sr4x5d\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001041520\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-vvj692\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-sr4x5d\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"fb349321ada687c5225f729f8d23b386912ad3c38f8e27ffdabb3962cf2c4359","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509681468542976","occurred_at":"2026-03-16T11:44:23Z","failed_at":"e2e","signature_id":"cc5128d498c1a9d710e6ba4aae837126e62dad94b3ea8136d4a71cdb77dc5bb4","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000f1c5c0\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-x658kt\" to finish creating: GET https://rp.j8542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c1bf58d2-c7a2-4659-9ac0-e943ec8118c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c1bf58d2-c7a2-4659-9ac0-e943ec8118c5\",\n \"name\": \"c1bf58d2-c7a2-4659-9ac0-e943ec8118c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:36:58.647009289Z\",\n \"endTime\": \"2026-03-16T12:38:12.224802026Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-104/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-104/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-x658kt\\\" to finish creating: GET https://rp.j8542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c1bf58d2-c7a2-4659-9ac0-e943ec8118c5\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c1bf58d2-c7a2-4659-9ac0-e943ec8118c5\\\",\\n \\\"name\\\": \\\"c1bf58d2-c7a2-4659-9ac0-e943ec8118c5\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:36:58.647009289Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:38:12.224802026Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-104/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-104/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000f1c580\u003e{\n msg: \"failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-x658kt\\\" to finish creating: GET https://rp.j8542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c1bf58d2-c7a2-4659-9ac0-e943ec8118c5\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c1bf58d2-c7a2-4659-9ac0-e943ec8118c5\\\",\\n \\\"name\\\": \\\"c1bf58d2-c7a2-4659-9ac0-e943ec8118c5\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:36:58.647009289Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:38:12.224802026Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-104/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-104/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrot...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000f1c5c0\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-x658kt\" to finish creating: GET https://rp.j8542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c1bf58d2-c7a2-4659-9ac0-e943ec8118c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c1bf58d2-c7a2-4659-9ac0-e943ec8118c5\",\n \"name\": \"c1bf58d2-c7a2-4659-9ac0-e943ec8118c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:36:58.647009289Z\",\n \"endTime\": \"2026-03-16T12:38:12.224802026Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-104/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-104/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-x658kt\\\" to finish creating: GET https://rp.j8542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c1bf58d2-c7a2-4659-9ac0-e943ec8118c5\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c1bf58d2-c7a2-4659-9ac0-e943ec8118c5\\\",\\n \\\"name\\\": \\\"c1bf58d2-c7a2-4659-9ac0-e943ec8118c5\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:36:58.647009289Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:38:12.224802026Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-104/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-104/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000f1c580\u003e{\n msg: \"failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-x658kt\\\" to finish creating: GET https://rp.j8542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c1bf58d2-c7a2-4659-9ac0-e943ec8118c5\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c1bf58d2-c7a2-4659-9ac0-e943ec8118c5\\\",\\n \\\"name\\\": \\\"c1bf58d2-c7a2-4659-9ac0-e943ec8118c5\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:36:58.647009289Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:38:12.224802026Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-104/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-104/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrot...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":11,"failure_pattern_ids":["3f672ce590a3e36623ba27892eef00b6b866047042ba8ed4eaf2c503e528aee8","503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","d55c4a29bbbf19a43335bfa2157018ebc82a1c8fe8cbea22f78ef2ca4c570f1a","fd93f6786bdbe9aeb08610dcc4ec01a349d1a8e658826e9448f8193966d5a67f"],"matched_occurrences":11,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4396/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509479131123712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4396,"pr_state":"closed","pr_sha":"a909a0005404923d9cff09140e0b188e374c1ee6","final_merged_sha":"732c1753890fbef90e84268733f7e4d18f32e691","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:43:35Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3dd0acdc3e7b6c8d237218d589d06ecf2488a71c62250fe90d9c3eb47087c0fb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4396/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509479131123712","occurred_at":"2026-03-16T11:43:35Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-16T12:04:32.942Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-16T12:04:35.189Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-16T12:04:36.549Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=2ce6fd3928f382c8dddd47ceca6425de2f0fccdb442dfd6d7e3753e4871af3f7\ntime=2026-03-16T12:06:07.148Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j1123712/providers/Microsoft.Resources/deployments/2ce6fd3928f382c8dddd47ceca6425de2f0fccdb442dfd6d7e3753e4871af3f7/operationStatuses/08584279442097117108\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1123712/providers/Microsoft.Resources/deployments/svc-monitor-ingestion-limits\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"InvalidRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The current utilization does not meet the criteria for both MaxTimeSeries and MaxEventsPerMinute quota requested. Please reach the required usage threshold of 50% of desired limit before requesting an increase, or request a limit increase of up to 200% of your current usage. For more details, see https://go.microsoft.com/fwlink/?linkid=2270124\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1123712/providers/Microsoft.Resources/deployments/hcp-monitor-ingestion-limits\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"InvalidRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The current utilization does not meet the criteria for both MaxTimeSeries and MaxEventsPerMinute quota requested. Please reach the required usage threshold of 50% of desired limit before requesting an increase, or request a limit increase of up to 200% of your current usage. For more details, see https://go.microsoft.com/fwlink/?linkid=2270124\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"090042207a3e0963476cf671303a807ca9b3b684acb44baf09806ab99342da0c","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InvalidRequest; detail message The current utilization does not meet the criteria for both MaxTimeSeries and MaxEventsPerMinute quota requested. Please reach the required usage threshold of 50% of desired limit before requesting an increase, or request a limit increase of up to 200% of your current usage.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["090042207a3e0963476cf671303a807ca9b3b684acb44baf09806ab99342da0c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4398/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509378581073920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4398,"pr_state":"closed","pr_sha":"06408b44aade23d9c2a20924d1d27f58cbb8ce19","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:43:11Z"},"failed_at":["e2e"],"failed_test_count":13,"occurrences":[{"row_id":"0394c728da77f05c4ed0b986c99d53cb4095398f15dfa193e0a96fb18e82ec4f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4398/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509378581073920","occurred_at":"2026-03-16T11:43:11Z","failed_at":"e2e","signature_id":"bce8e003356d6224c575df9fddd2cc7e9ed76ac2df579c92243c006ddb066f4d","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000dac9d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-l9gr75\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000e177c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-l9gr75\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e177a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-l9gr75\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e17780\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-l9gr75\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000dac9d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-l9gr75\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000e177c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-l9gr75\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e177a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-l9gr75\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e17780\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-l9gr75\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"08cd33e7dff49b93f89157da317b930fd9c4433f3a27a1bfb998d145b76a1982","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4398/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509378581073920","occurred_at":"2026-03-16T11:43:11Z","failed_at":"e2e","signature_id":"6157917f602ae6d5aed152f7579ec41d6ed926846e440b2238eaadcffa8c6b1c","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:86]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001766b20\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-7dvc44\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/437e5eb7-6380-44d3-a27a-311c39395c6f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/437e5eb7-6380-44d3-a27a-311c39395c6f\",\n \"name\": \"437e5eb7-6380-44d3-a27a-311c39395c6f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:56:49.020928274Z\",\n \"endTime\": \"2026-03-16T12:57:52.704479743Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-103/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-103/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-7dvc44\\\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/437e5eb7-6380-44d3-a27a-311c39395c6f\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/437e5eb7-6380-44d3-a27a-311c39395c6f\\\",\\n \\\"name\\\": \\\"437e5eb7-6380-44d3-a27a-311c39395c6f\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:56:49.020928274Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:57:52.704479743Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-103/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-103/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001766b00\u003e{\n msg: \"failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-7dvc44\\\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/437e5eb7-6380-44d3-a27a-311c39395c6f\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/437e5eb7-6380-44d3-a27a-311c39395c6f\\\",\\n \\\"name\\\": \\\"437e5eb7-6380-44d3-a27a-311c39395c6f\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:56:49.020928274Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:57:52.704479743Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-103/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-103/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\":...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:86]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001766b20\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-7dvc44\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/437e5eb7-6380-44d3-a27a-311c39395c6f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/437e5eb7-6380-44d3-a27a-311c39395c6f\",\n \"name\": \"437e5eb7-6380-44d3-a27a-311c39395c6f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:56:49.020928274Z\",\n \"endTime\": \"2026-03-16T12:57:52.704479743Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-103/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-103/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-7dvc44\\\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/437e5eb7-6380-44d3-a27a-311c39395c6f\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/437e5eb7-6380-44d3-a27a-311c39395c6f\\\",\\n \\\"name\\\": \\\"437e5eb7-6380-44d3-a27a-311c39395c6f\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:56:49.020928274Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:57:52.704479743Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-103/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-103/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001766b00\u003e{\n msg: \"failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-7dvc44\\\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/437e5eb7-6380-44d3-a27a-311c39395c6f\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/437e5eb7-6380-44d3-a27a-311c39395c6f\\\",\\n \\\"name\\\": \\\"437e5eb7-6380-44d3-a27a-311c39395c6f\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:56:49.020928274Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:57:52.704479743Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-103/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-103/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\":...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"64b64d441de0a499c47f1ee75928223d7fcef9aa070126f81f7f011ec3ff0aed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4398/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509378581073920","occurred_at":"2026-03-16T11:43:11Z","failed_at":"e2e","signature_id":"e1f3838548c7e458e1c31750150060ab3e5cb1dd210979771aa1cf79a088a294","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00091e220\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-mrc89s\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/465467ce-7ab5-4614-a7c0-3f3868589f0c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/465467ce-7ab5-4614-a7c0-3f3868589f0c\",\n \"name\": \"465467ce-7ab5-4614-a7c0-3f3868589f0c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:35:08.997993551Z\",\n \"endTime\": \"2026-03-16T12:36:36.827564904Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-106/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-106/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-mrc89s\\\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/465467ce-7ab5-4614-a7c0-3f3868589f0c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/465467ce-7ab5-4614-a7c0-3f3868589f0c\\\",\\n \\\"name\\\": \\\"465467ce-7ab5-4614-a7c0-3f3868589f0c\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:35:08.997993551Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:36:36.827564904Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-106/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-106/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00091e200\u003e{\n msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-mrc89s\\\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/465467ce-7ab5-4614-a7c0-3f3868589f0c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/465467ce-7ab5-4614-a7c0-3f3868589f0c\\\",\\n \\\"name\\\": \\\"465467ce-7ab5-4614-a7c0-3f3868589f0c\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:35:08.997993551Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:36:36.827564904Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-106/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-106/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00091e220\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-mrc89s\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/465467ce-7ab5-4614-a7c0-3f3868589f0c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/465467ce-7ab5-4614-a7c0-3f3868589f0c\",\n \"name\": \"465467ce-7ab5-4614-a7c0-3f3868589f0c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:35:08.997993551Z\",\n \"endTime\": \"2026-03-16T12:36:36.827564904Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-106/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-106/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-mrc89s\\\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/465467ce-7ab5-4614-a7c0-3f3868589f0c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/465467ce-7ab5-4614-a7c0-3f3868589f0c\\\",\\n \\\"name\\\": \\\"465467ce-7ab5-4614-a7c0-3f3868589f0c\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:35:08.997993551Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:36:36.827564904Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-106/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-106/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00091e200\u003e{\n msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-mrc89s\\\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/465467ce-7ab5-4614-a7c0-3f3868589f0c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/465467ce-7ab5-4614-a7c0-3f3868589f0c\\\",\\n \\\"name\\\": \\\"465467ce-7ab5-4614-a7c0-3f3868589f0c\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:35:08.997993551Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:36:36.827564904Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-106/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-106/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6a160cb9dfe63e32d96139a6ca7325deb35827681bf32f6bc3d268a9ff84e43c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4398/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509378581073920","occurred_at":"2026-03-16T11:43:11Z","failed_at":"e2e","signature_id":"ab1120b242d24709cf58cda03f74893bdac42cfb30aef1c9260418f61f9c14ec","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00116e1b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-jkvnzz\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0006fca00\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-jkvnzz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006fc9c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-jkvnzz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006fc880\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-jkvnzz\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00116e1b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-jkvnzz\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0006fca00\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-jkvnzz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006fc9c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-jkvnzz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006fc880\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-jkvnzz\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"7633132212b395c9a723496a4a0c5972f7ea8afc6574d16b1363fc7697fd26bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4398/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509378581073920","occurred_at":"2026-03-16T11:43:11Z","failed_at":"e2e","signature_id":"acb411f31018cdf221417c4efeb50c8a738fd3bdf39b161b940679de242434b5","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000c7e3d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-xl2xk8\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0014d84e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-xl2xk8\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0014d84c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-xl2xk8\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0014d84a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-xl2xk8\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000c7e3d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-xl2xk8\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0014d84e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-xl2xk8\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0014d84c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-xl2xk8\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0014d84a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-xl2xk8\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"80a118ef6aa73a5335f33bd93a7231e100f31559b959fd22cb5c7aa5b1023b42","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4398/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509378581073920","occurred_at":"2026-03-16T11:43:11Z","failed_at":"e2e","signature_id":"5e1d1cecef0e3ba871bd508cb1cc1d64267db7c306cba44ddc177a940a5761a0","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b85c20\u003e: \n failed to create HCP cluster cluster-zstream-4-20-6r5wcr: failed waiting for cluster=\"cluster-zstream-4-20-6r5wcr\" in resourcegroup=\"rg-zstream-upgrade-4-20-8cs2kg\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/066bfaeb-683b-415d-afa1-18373f3e2bfb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/066bfaeb-683b-415d-afa1-18373f3e2bfb\",\n \"name\": \"066bfaeb-683b-415d-afa1-18373f3e2bfb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:39:05.722653538Z\",\n \"endTime\": \"2026-03-16T12:40:27.237556224Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-124/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-124/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-20-6r5wcr: failed waiting for cluster=\\\"cluster-zstream-4-20-6r5wcr\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-8cs2kg\\\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/066bfaeb-683b-415d-afa1-18373f3e2bfb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/066bfaeb-683b-415d-afa1-18373f3e2bfb\\\",\\n \\\"name\\\": \\\"066bfaeb-683b-415d-afa1-18373f3e2bfb\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:39:05.722653538Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:40:27.237556224Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-124/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-124/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000b85c00\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-6r5wcr\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-8cs2kg\\\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/066bfaeb-683b-415d-afa1-18373f3e2bfb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/066bfaeb-683b-415d-afa1-18373f3e2bfb\\\",\\n \\\"name\\\": \\\"066bfaeb-683b-415d-afa1-18373f3e2bfb\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:39:05.722653538Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:40:27.237556224Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-124/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-124/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-----------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b85c20\u003e: \n failed to create HCP cluster cluster-zstream-4-20-6r5wcr: failed waiting for cluster=\"cluster-zstream-4-20-6r5wcr\" in resourcegroup=\"rg-zstream-upgrade-4-20-8cs2kg\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/066bfaeb-683b-415d-afa1-18373f3e2bfb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/066bfaeb-683b-415d-afa1-18373f3e2bfb\",\n \"name\": \"066bfaeb-683b-415d-afa1-18373f3e2bfb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:39:05.722653538Z\",\n \"endTime\": \"2026-03-16T12:40:27.237556224Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-124/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-124/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-20-6r5wcr: failed waiting for cluster=\\\"cluster-zstream-4-20-6r5wcr\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-8cs2kg\\\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/066bfaeb-683b-415d-afa1-18373f3e2bfb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/066bfaeb-683b-415d-afa1-18373f3e2bfb\\\",\\n \\\"name\\\": \\\"066bfaeb-683b-415d-afa1-18373f3e2bfb\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:39:05.722653538Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:40:27.237556224Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-124/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-124/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000b85c00\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-6r5wcr\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-8cs2kg\\\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/066bfaeb-683b-415d-afa1-18373f3e2bfb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/066bfaeb-683b-415d-afa1-18373f3e2bfb\\\",\\n \\\"name\\\": \\\"066bfaeb-683b-415d-afa1-18373f3e2bfb\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:39:05.722653538Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:40:27.237556224Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-124/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-124/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-----------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a283b1f16a3408638752c849473d3ae7a352a19bc1f80aef5b1860dd7f9b2cba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4398/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509378581073920","occurred_at":"2026-03-16T11:43:11Z","failed_at":"e2e","signature_id":"0cc3d16d47ebcfc3d74a0653611836df8bcf989f3e5052aea43c8c6950ccd8d0","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e322b8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-822n24\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000d8d3c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-822n24\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d8d3a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-822n24\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d8d380\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-822n24\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e322b8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-822n24\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000d8d3c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-822n24\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d8d3a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-822n24\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d8d380\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-822n24\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"b20f1f0f505a17cd61361e55e49474119e8c36419e562365c81c0acd30c577a4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4398/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509378581073920","occurred_at":"2026-03-16T11:43:11Z","failed_at":"e2e","signature_id":"15c42c96f1546b0047d014d2b1c6aea88bb17c6518e5bac8812aa923f86d64c2","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:89]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000f12680\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-tnp5xw\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e8b8c903-1c44-431d-90cc-722e05f4d5b6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e8b8c903-1c44-431d-90cc-722e05f4d5b6\",\n \"name\": \"e8b8c903-1c44-431d-90cc-722e05f4d5b6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:34:33.833275458Z\",\n \"endTime\": \"2026-03-16T12:35:37.744141088Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-124/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-124/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-tnp5xw\\\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e8b8c903-1c44-431d-90cc-722e05f4d5b6\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e8b8c903-1c44-431d-90cc-722e05f4d5b6\\\",\\n \\\"name\\\": \\\"e8b8c903-1c44-431d-90cc-722e05f4d5b6\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:34:33.833275458Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:35:37.744141088Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-124/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-124/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000f125e0\u003e{\n msg: \"failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-tnp5xw\\\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e8b8c903-1c44-431d-90cc-722e05f4d5b6\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e8b8c903-1c44-431d-90cc-722e05f4d5b6\\\",\\n \\\"name\\\": \\\"e8b8c903-1c44-431d-90cc-722e05f4d5b6\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:34:33.833275458Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:35:37.744141088Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-124/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-124/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:89]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000f12680\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-tnp5xw\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e8b8c903-1c44-431d-90cc-722e05f4d5b6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e8b8c903-1c44-431d-90cc-722e05f4d5b6\",\n \"name\": \"e8b8c903-1c44-431d-90cc-722e05f4d5b6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:34:33.833275458Z\",\n \"endTime\": \"2026-03-16T12:35:37.744141088Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-124/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-124/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-tnp5xw\\\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e8b8c903-1c44-431d-90cc-722e05f4d5b6\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e8b8c903-1c44-431d-90cc-722e05f4d5b6\\\",\\n \\\"name\\\": \\\"e8b8c903-1c44-431d-90cc-722e05f4d5b6\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:34:33.833275458Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:35:37.744141088Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-124/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-124/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000f125e0\u003e{\n msg: \"failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-tnp5xw\\\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e8b8c903-1c44-431d-90cc-722e05f4d5b6\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e8b8c903-1c44-431d-90cc-722e05f4d5b6\\\",\\n \\\"name\\\": \\\"e8b8c903-1c44-431d-90cc-722e05f4d5b6\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:34:33.833275458Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:35:37.744141088Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-124/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-124/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c19ab148d5c765436f9df129e5fd2a5f9a677187ccec8f31fff218410140b247","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4398/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509378581073920","occurred_at":"2026-03-16T11:43:11Z","failed_at":"e2e","signature_id":"cbfc03aa71b72b834ab4ae666ea0575fd42d5edbcf4f9866959be24afda432c5","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000532660\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-22btj8\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86169469-3c78-4d45-a751-69b83d198d83\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86169469-3c78-4d45-a751-69b83d198d83\",\n \"name\": \"86169469-3c78-4d45-a751-69b83d198d83\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:56:13.574728771Z\",\n \"endTime\": \"2026-03-16T12:56:58.652053428Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-135/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-135/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-22btj8\\\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86169469-3c78-4d45-a751-69b83d198d83\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86169469-3c78-4d45-a751-69b83d198d83\\\",\\n \\\"name\\\": \\\"86169469-3c78-4d45-a751-69b83d198d83\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:56:13.574728771Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:56:58.652053428Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-135/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-135/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000532640\u003e{\n msg: \"failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-22btj8\\\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86169469-3c78-4d45-a751-69b83d198d83\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86169469-3c78-4d45-a751-69b83d198d83\\\",\\n \\\"name\\\": \\\"86169469-3c78-4d45-a751-69b83d198d83\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:56:13.574728771Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:56:58.652053428Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-135/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-135/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000532660\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-22btj8\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86169469-3c78-4d45-a751-69b83d198d83\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86169469-3c78-4d45-a751-69b83d198d83\",\n \"name\": \"86169469-3c78-4d45-a751-69b83d198d83\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:56:13.574728771Z\",\n \"endTime\": \"2026-03-16T12:56:58.652053428Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-135/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-135/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-22btj8\\\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86169469-3c78-4d45-a751-69b83d198d83\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86169469-3c78-4d45-a751-69b83d198d83\\\",\\n \\\"name\\\": \\\"86169469-3c78-4d45-a751-69b83d198d83\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:56:13.574728771Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:56:58.652053428Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-135/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-135/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000532640\u003e{\n msg: \"failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-22btj8\\\" to finish creating: GET https://rp.j1073920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86169469-3c78-4d45-a751-69b83d198d83\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86169469-3c78-4d45-a751-69b83d198d83\\\",\\n \\\"name\\\": \\\"86169469-3c78-4d45-a751-69b83d198d83\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:56:13.574728771Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:56:58.652053428Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-135/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-135/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c6be2a6af39549ace14bb8bbafff268b568ddcd22bed3922cf9539fbd5c33d5c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4398/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509378581073920","occurred_at":"2026-03-16T11:43:11Z","failed_at":"e2e","signature_id":"952c666346d31d95303fc2046bee3cd9963d18aa11abf030194160c16fea5cdc","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0016ce180\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-ldh25c\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000b075c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-ldh25c\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b07580\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-ldh25c\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b07560\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-ldh25c\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0016ce180\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-ldh25c\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000b075c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-ldh25c\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b07580\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-ldh25c\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b07560\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-ldh25c\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"c92744663eddf9837728e9ce583fcec956cadb154a7f5b9996b1fed895a5eff5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4398/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509378581073920","occurred_at":"2026-03-16T11:43:11Z","failed_at":"e2e","signature_id":"f961192b4451a96867f085bdc3888c1af2884be6946a0044308725e5beadb12c","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e00120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-8khv2k\" in resourcegroup=\"rg-zstream-upgrade-4-21-bxzggv\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000759a20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-8khv2k\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-bxzggv\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000759900\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-8khv2k\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-bxzggv\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007597c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-8khv2k\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-bxzggv\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e00120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-8khv2k\" in resourcegroup=\"rg-zstream-upgrade-4-21-bxzggv\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000759a20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-8khv2k\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-bxzggv\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000759900\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-8khv2k\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-bxzggv\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007597c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-8khv2k\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-bxzggv\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"eb736d27d70c82c8c572b968bf85e01d872e4576b1c3c68c5d620fd68e1e1c32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4398/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509378581073920","occurred_at":"2026-03-16T11:43:11Z","failed_at":"e2e","signature_id":"8e6c799fcc780014df5302843ef7e855a08c40cae9b9198a307d42f7087fef85","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00108c318\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-c7m2z6\" in resourcegroup=\"admin-credential-lifecycle-test-2kddvf\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000805680\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-c7m2z6\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-2kddvf\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000805660\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-c7m2z6\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-2kddvf\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000805640\u003e{\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-c7m2z6\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-2kddvf\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00108c318\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-c7m2z6\" in resourcegroup=\"admin-credential-lifecycle-test-2kddvf\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000805680\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-c7m2z6\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-2kddvf\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000805660\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-c7m2z6\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-2kddvf\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000805640\u003e{\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-c7m2z6\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-2kddvf\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"fd915d5bad7d8aad4151de1cc53d5320cf0991bfde90f47f9ccb92bbd3f9778b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4398/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509378581073920","occurred_at":"2026-03-16T11:43:11Z","failed_at":"e2e","signature_id":"193cdb3f563bac1915ba5938ed99724b744d7745313fb35d63d8689e14644be3","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ed4330\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-vlv4fk\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001341160\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-vlv4fk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001341140\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-vlv4fk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001341120\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-vlv4fk\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ed4330\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-vlv4fk\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001341160\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-vlv4fk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001341140\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-vlv4fk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001341120\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-vlv4fk\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":13,"failure_pattern_ids":["503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761"],"matched_occurrences":13,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4402/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509277980692480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4402,"pr_state":"closed","pr_sha":"fca3d7b848ee68f6eef74b75550b94f0462c3329","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:42:47Z"},"failed_at":["e2e"],"failed_test_count":16,"occurrences":[{"row_id":"1a8f9a40ffaccea2a16742b4f3515212a3fd6873aa7d06a1fd8b2aad1925318a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4402/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509277980692480","occurred_at":"2026-03-16T11:42:47Z","failed_at":"e2e","signature_id":"4ee4b6691411e5d3623ee8e0c3750dd47af71932f38419fedbde04be081c6a5f","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f0e1c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-p7rzwx\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0011a10c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-p7rzwx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0011a10a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-p7rzwx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0011a1080\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-p7rzwx\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f0e1c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-p7rzwx\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0011a10c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-p7rzwx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0011a10a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-p7rzwx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0011a1080\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-p7rzwx\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"1b309bd7728749b6092e9297f3d98c70f4d475cb726855c5ebe3e8fece287843","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4402/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509277980692480","occurred_at":"2026-03-16T11:42:47Z","failed_at":"e2e","signature_id":"8c80e8ac3fa6706a1d6a8721504938bd9a9ff1085f993abc089d5a3bef7d72e1","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000de57a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-v29lpc\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00141cd40\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-v29lpc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00141cd20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-v29lpc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00141cd00\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-v29lpc\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000de57a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-v29lpc\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00141cd40\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-v29lpc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00141cd20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-v29lpc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00141cd00\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-v29lpc\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"1b6e577f6c690e6a3ebf71b8cc4308e3ab96ec0fe4df03e86139c5dcdb685561","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4402/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509277980692480","occurred_at":"2026-03-16T11:42:47Z","failed_at":"e2e","signature_id":"bbda93946c12bc4a476b11db38acba56bb0aa1e0d800b31247d6a04baf635704","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e14150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-b9mmsd\" in resourcegroup=\"rg-zstream-upgrade-4-21-hwl84h\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000797880\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-b9mmsd\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-hwl84h\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000797860\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-b9mmsd\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-hwl84h\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000797840\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-b9mmsd\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-hwl84h\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e14150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-b9mmsd\" in resourcegroup=\"rg-zstream-upgrade-4-21-hwl84h\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000797880\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-b9mmsd\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-hwl84h\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000797860\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-b9mmsd\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-hwl84h\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000797840\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-b9mmsd\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-hwl84h\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"2e838242b25df087d641a2b5a7e585e3dcab303d15001ddba40eec248089a2a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4402/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509277980692480","occurred_at":"2026-03-16T11:42:47Z","failed_at":"e2e","signature_id":"dd0b5a26974ff0f56ab1aede9eec8db20185bd5f5bcc346cbc5ab99aec2b0f5f","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ec02b8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-lvrdg9\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0013eb2c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-lvrdg9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0013eb2a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-lvrdg9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0013eb280\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-lvrdg9\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ec02b8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-lvrdg9\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0013eb2c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-lvrdg9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0013eb2a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-lvrdg9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0013eb280\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-lvrdg9\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"31a4268f2a8741d655b27f63dbcd2ced63e4f3e93414fbd93d1cd1d08352b0a1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4402/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509277980692480","occurred_at":"2026-03-16T11:42:47Z","failed_at":"e2e","signature_id":"33537f6bdcbee4f38900ea8ede529257bd41e622065aa0f0a59324c60990f9d9","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00172e1e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-7zq2cq\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007818a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-7zq2cq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000781880\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-7zq2cq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000781860\u003e{\n msg: \"failed waiting for hcpCluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-7zq2cq\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00172e1e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-7zq2cq\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007818a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-7zq2cq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000781880\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-7zq2cq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000781860\u003e{\n msg: \"failed waiting for hcpCluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-7zq2cq\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"3334ac1815e5b1e8150b440c0623f5b1c346c77842416fd4238fc111e9b0417c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4402/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509277980692480","occurred_at":"2026-03-16T11:42:47Z","failed_at":"e2e","signature_id":"231667ec12102509c60179a433daf67489fbcd3f7b40b90ae47b24f6173b07fc","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0014dc150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-hhv557\" in resourcegroup=\"rg-zstream-upgrade-4-20-rrxqrj\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000e9cea0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-hhv557\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-rrxqrj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e9ce80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-hhv557\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-rrxqrj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e9ce60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-20-hhv557\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-rrxqrj\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0014dc150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-hhv557\" in resourcegroup=\"rg-zstream-upgrade-4-20-rrxqrj\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000e9cea0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-hhv557\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-rrxqrj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e9ce80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-hhv557\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-rrxqrj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e9ce60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-20-hhv557\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-rrxqrj\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"7e3aa01770d1fdab56db14b98266abe6787588f2e73e58fbffc6a43acbdcc7e1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4402/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509277980692480","occurred_at":"2026-03-16T11:42:47Z","failed_at":"e2e","signature_id":"691cb3db30c010df155597938ee771a337ab44436d04446f9dd57222bca3e799","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00162a1b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-qbt2j4\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000970200\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-qbt2j4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009701e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-qbt2j4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009701c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-qbt2j4\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00162a1b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-qbt2j4\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000970200\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-qbt2j4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009701e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-qbt2j4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009701c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-qbt2j4\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"81cc0b1bdb77bc62f5c808517652d6bc6f4742493fa53fbc2be688fabfd3b7c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4402/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509277980692480","occurred_at":"2026-03-16T11:42:47Z","failed_at":"e2e","signature_id":"8c6ba0b8ad4208016fa6a3b7dd14047af841a2f0acc58f0c0e321e5ea18c90c6","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e22140\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-7h22r5\" to finish creating: GET https://rp.j0692480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6b562372-8e68-442b-a67b-d54e8043f45c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6b562372-8e68-442b-a67b-d54e8043f45c\",\n \"name\": \"6b562372-8e68-442b-a67b-d54e8043f45c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:32:23.627486385Z\",\n \"endTime\": \"2026-03-16T12:33:16.905887614Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-266/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-266/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-7h22r5\\\" to finish creating: GET https://rp.j0692480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6b562372-8e68-442b-a67b-d54e8043f45c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6b562372-8e68-442b-a67b-d54e8043f45c\\\",\\n \\\"name\\\": \\\"6b562372-8e68-442b-a67b-d54e8043f45c\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:32:23.627486385Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:33:16.905887614Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-266/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-266/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e22120\u003e{\n msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-7h22r5\\\" to finish creating: GET https://rp.j0692480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6b562372-8e68-442b-a67b-d54e8043f45c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6b562372-8e68-442b-a67b-d54e8043f45c\\\",\\n \\\"name\\\": \\\"6b562372-8e68-442b-a67b-d54e8043f45c\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:32:23.627486385Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:33:16.905887614Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-266/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-266/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e22140\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-7h22r5\" to finish creating: GET https://rp.j0692480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6b562372-8e68-442b-a67b-d54e8043f45c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6b562372-8e68-442b-a67b-d54e8043f45c\",\n \"name\": \"6b562372-8e68-442b-a67b-d54e8043f45c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:32:23.627486385Z\",\n \"endTime\": \"2026-03-16T12:33:16.905887614Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-266/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-266/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-7h22r5\\\" to finish creating: GET https://rp.j0692480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6b562372-8e68-442b-a67b-d54e8043f45c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6b562372-8e68-442b-a67b-d54e8043f45c\\\",\\n \\\"name\\\": \\\"6b562372-8e68-442b-a67b-d54e8043f45c\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:32:23.627486385Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:33:16.905887614Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-266/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-266/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e22120\u003e{\n msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-7h22r5\\\" to finish creating: GET https://rp.j0692480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6b562372-8e68-442b-a67b-d54e8043f45c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6b562372-8e68-442b-a67b-d54e8043f45c\\\",\\n \\\"name\\\": \\\"6b562372-8e68-442b-a67b-d54e8043f45c\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:32:23.627486385Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:33:16.905887614Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-266/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-266/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"829ddac3f72f31d7f02da9d52eb67ad7ab647f481d2f6c7665c87907ab253273","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4402/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509277980692480","occurred_at":"2026-03-16T11:42:47Z","failed_at":"e2e","signature_id":"0a6191fe441cbdd3cc43d502be86964eda37c212c309297e6b78cbe571e27ba0","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0008bcd98\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-rrgcjp\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a1fb00\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-rrgcjp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a1fac0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-rrgcjp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a1faa0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-rrgcjp\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0008bcd98\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-rrgcjp\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a1fb00\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-rrgcjp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a1fac0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-rrgcjp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a1faa0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-rrgcjp\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"97bfecb03043ad60c4190102ab33389bab5dc1bf9f050f5ba90b477d845e5202","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4402/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509277980692480","occurred_at":"2026-03-16T11:42:47Z","failed_at":"e2e","signature_id":"fc49c570e92c895b32c0982d060759acdc6a9603455835c7b0e41fbb0783152e","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d90108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-n9zq8q\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007737e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-n9zq8q\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007737c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-n9zq8q\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007737a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-n9zq8q\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d90108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-n9zq8q\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007737e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-n9zq8q\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007737c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-n9zq8q\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007737a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-n9zq8q\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"ab7a2cc24c9a29233c126b76166fa9ffcbbf330826b8b623bb1a43936ac4b48d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4402/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509277980692480","occurred_at":"2026-03-16T11:42:47Z","failed_at":"e2e","signature_id":"f69d692f65cc5bfc73eb448b4dbf29e9b86602d4d816a7d77289de760954e5de","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000772ec0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-nh4s5b\" to finish creating: GET https://rp.j0692480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6bc64492-e6a4-4fa8-bf06-17ab05a15549\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6bc64492-e6a4-4fa8-bf06-17ab05a15549\",\n \"name\": \"6bc64492-e6a4-4fa8-bf06-17ab05a15549\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:32:21.645989348Z\",\n \"endTime\": \"2026-03-16T12:33:16.903071324Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-245/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-245/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster ea-cluster: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-nh4s5b\\\" to finish creating: GET https://rp.j0692480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6bc64492-e6a4-4fa8-bf06-17ab05a15549\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6bc64492-e6a4-4fa8-bf06-17ab05a15549\\\",\\n \\\"name\\\": \\\"6bc64492-e6a4-4fa8-bf06-17ab05a15549\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:32:21.645989348Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:33:16.903071324Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-245/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-245/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000772e40\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-nh4s5b\\\" to finish creating: GET https://rp.j0692480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6bc64492-e6a4-4fa8-bf06-17ab05a15549\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6bc64492-e6a4-4fa8-bf06-17ab05a15549\\\",\\n \\\"name\\\": \\\"6bc64492-e6a4-4fa8-bf06-17ab05a15549\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:32:21.645989348Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:33:16.903071324Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-245/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-245/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000772ec0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-nh4s5b\" to finish creating: GET https://rp.j0692480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6bc64492-e6a4-4fa8-bf06-17ab05a15549\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6bc64492-e6a4-4fa8-bf06-17ab05a15549\",\n \"name\": \"6bc64492-e6a4-4fa8-bf06-17ab05a15549\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:32:21.645989348Z\",\n \"endTime\": \"2026-03-16T12:33:16.903071324Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-245/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-245/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster ea-cluster: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-nh4s5b\\\" to finish creating: GET https://rp.j0692480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6bc64492-e6a4-4fa8-bf06-17ab05a15549\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6bc64492-e6a4-4fa8-bf06-17ab05a15549\\\",\\n \\\"name\\\": \\\"6bc64492-e6a4-4fa8-bf06-17ab05a15549\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:32:21.645989348Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:33:16.903071324Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-245/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-245/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000772e40\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-nh4s5b\\\" to finish creating: GET https://rp.j0692480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6bc64492-e6a4-4fa8-bf06-17ab05a15549\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6bc64492-e6a4-4fa8-bf06-17ab05a15549\\\",\\n \\\"name\\\": \\\"6bc64492-e6a4-4fa8-bf06-17ab05a15549\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:32:21.645989348Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:33:16.903071324Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-245/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-245/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b239fec10fb967012d0bb08ac4c8bf10e4d64a21eb42eeeee9e57e55e47cca73","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4402/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509277980692480","occurred_at":"2026-03-16T11:42:47Z","failed_at":"e2e","signature_id":"06e621cd9e9f69ef7d22ffa143e6ffdd58367377323e8b6a294d4a7548d7bb5a","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00082a360\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-l27lxr\" in resourcegroup=\"admin-credential-lifecycle-test-kxjv9r\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00071de80\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-l27lxr\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-kxjv9r\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00071de20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-l27lxr\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-kxjv9r\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00071dde0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-l27lxr\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-kxjv9r\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00082a360\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-l27lxr\" in resourcegroup=\"admin-credential-lifecycle-test-kxjv9r\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00071de80\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-l27lxr\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-kxjv9r\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00071de20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-l27lxr\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-kxjv9r\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00071dde0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-l27lxr\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-kxjv9r\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"c1e5f1bf1913ec642535ed76d1182fc3011fab39015aed2cdeaf60feb0fa84a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4402/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509277980692480","occurred_at":"2026-03-16T11:42:47Z","failed_at":"e2e","signature_id":"275ae94dded5ecbeff89c6f98a68c813082dca77a93f3d6ffe20398e5db303ab","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0004aa2d0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-h24hxm\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000813900\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-h24hxm\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008138e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-h24hxm\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008138c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-h24hxm\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0004aa2d0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-h24hxm\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000813900\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-h24hxm\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008138e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-h24hxm\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008138c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-h24hxm\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"c9b738dc37f98c0d5910157c6ed7956166a48d9aadc180d34cb22a1adc809b27","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4402/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509277980692480","occurred_at":"2026-03-16T11:42:47Z","failed_at":"e2e","signature_id":"e04d6d3613638ede3042ae7db3d486cef66b72a2cb1684898db07dbe48d70606","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0009646a0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-tdsjqt\" to finish creating: GET https://rp.j0692480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c4fe4c6-eac0-470b-b2fb-9df4cab39e5d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c4fe4c6-eac0-470b-b2fb-9df4cab39e5d\",\n \"name\": \"7c4fe4c6-eac0-470b-b2fb-9df4cab39e5d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:34:43.178140778Z\",\n \"endTime\": \"2026-03-16T12:36:17.674286842Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-tdsjqt/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-tdsjqt\\\" to finish creating: GET https://rp.j0692480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c4fe4c6-eac0-470b-b2fb-9df4cab39e5d\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c4fe4c6-eac0-470b-b2fb-9df4cab39e5d\\\",\\n \\\"name\\\": \\\"7c4fe4c6-eac0-470b-b2fb-9df4cab39e5d\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:34:43.178140778Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:36:17.674286842Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-tdsjqt/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000964680\u003e{\n msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-tdsjqt\\\" to finish creating: GET https://rp.j0692480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c4fe4c6-eac0-470b-b2fb-9df4cab39e5d\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c4fe4c6-eac0-470b-b2fb-9df4cab39e5d\\\",\\n \\\"name\\\": \\\"7c4fe4c6-eac0-470b-b2fb-9df4cab39e5d\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:34:43.178140778Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:36:17.674286842Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-tdsjqt/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0009646a0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-tdsjqt\" to finish creating: GET https://rp.j0692480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c4fe4c6-eac0-470b-b2fb-9df4cab39e5d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c4fe4c6-eac0-470b-b2fb-9df4cab39e5d\",\n \"name\": \"7c4fe4c6-eac0-470b-b2fb-9df4cab39e5d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:34:43.178140778Z\",\n \"endTime\": \"2026-03-16T12:36:17.674286842Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-tdsjqt/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-tdsjqt\\\" to finish creating: GET https://rp.j0692480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c4fe4c6-eac0-470b-b2fb-9df4cab39e5d\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c4fe4c6-eac0-470b-b2fb-9df4cab39e5d\\\",\\n \\\"name\\\": \\\"7c4fe4c6-eac0-470b-b2fb-9df4cab39e5d\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:34:43.178140778Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:36:17.674286842Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-tdsjqt/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000964680\u003e{\n msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-tdsjqt\\\" to finish creating: GET https://rp.j0692480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c4fe4c6-eac0-470b-b2fb-9df4cab39e5d\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c4fe4c6-eac0-470b-b2fb-9df4cab39e5d\\\",\\n \\\"name\\\": \\\"7c4fe4c6-eac0-470b-b2fb-9df4cab39e5d\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:34:43.178140778Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:36:17.674286842Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-tdsjqt/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d6f7039945123f5d58378663ba4e737ecbdca62a620625b25254fa813213dcc3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4402/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509277980692480","occurred_at":"2026-03-16T11:42:47Z","failed_at":"e2e","signature_id":"e85f8922245f281831aab7a1a315a1390a4baa668d99f447a15859fb7e9e504c","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010cc498\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-kkct5z\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a0d7a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-kkct5z\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a0d780\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-kkct5z\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a0d760\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-kkct5z\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010cc498\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-kkct5z\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a0d7a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-kkct5z\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a0d780\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-kkct5z\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a0d760\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-kkct5z\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"f3f2ecc2e4f0d446c41508c4cde58de048585feec4d06b38f0305f549bde6a1c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4402/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033509277980692480","occurred_at":"2026-03-16T11:42:47Z","failed_at":"e2e","signature_id":"ec4705c6df65b5d4737ed921a549ce096d343c6a93c0f204edb8c1476cf1058c","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00110a3a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-c7k8n8\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000fa9040\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-c7k8n8\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000fa9020\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-c7k8n8\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000fa9000\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-c7k8n8\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00110a3a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-c7k8n8\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000fa9040\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-c7k8n8\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000fa9020\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-c7k8n8\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000fa9000\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-c7k8n8\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":16,"failure_pattern_ids":["503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761"],"matched_occurrences":16,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508825784389632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4407,"pr_state":"closed","pr_sha":"edbec11fb398f9955d7bfb3c4306b4e339a434d3","final_merged_sha":"80403bbc3576c61abfcd5140651ca88e1350d2ac","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:40:59Z"},"failed_at":["e2e"],"failed_test_count":9,"occurrences":[{"row_id":"075aa1ce2fc657317fe01a95111ca3cebd2510e4d6b6b9ffba25512054816fb8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508825784389632","occurred_at":"2026-03-16T11:40:59Z","failed_at":"e2e","signature_id":"99f7fe6455d7998bd9a56414691c0f4cf308268bd07db06c9a04145ead227e29","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d12318\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-jz9nh4\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f60d60\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-jz9nh4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f60d40\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-jz9nh4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f60d20\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-jz9nh4\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d12318\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-jz9nh4\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f60d60\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-jz9nh4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f60d40\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-jz9nh4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f60d20\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-jz9nh4\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"688848cf327db1f7b160410408625fe5441e496fa1470f0e9fc3bb4a3a15807c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508825784389632","occurred_at":"2026-03-16T11:40:59Z","failed_at":"e2e","signature_id":"ae479e53c1674778bfbdba4df54cf494bf576f25e5f6264791c5861ac1451e7e","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000b92258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-7lnqj8\" in resourcegroup=\"rg-zstream-upgrade-4-21-z269g9\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000b2f7e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-7lnqj8\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-z269g9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b2f7c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-7lnqj8\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-z269g9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b2f7a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-7lnqj8\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-z269g9\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000b92258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-7lnqj8\" in resourcegroup=\"rg-zstream-upgrade-4-21-z269g9\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000b2f7e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-7lnqj8\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-z269g9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b2f7c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-7lnqj8\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-z269g9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b2f7a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-7lnqj8\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-z269g9\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"71fea0e0643edde35d92403231bd106e1c6cc766d26ea9e2494b355dc2187702","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508825784389632","occurred_at":"2026-03-16T11:40:59Z","failed_at":"e2e","signature_id":"f3e64efdd7cbf2797b38db0ab0bf9aa964a7ae954b8fa3c7be2ecd25740716a6","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:73]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0013642e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-update-nodes-hcp-cluster-q4vvzb\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-q4vvzb/operationStatuses/08584279343875914687\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-vtr5fg/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-214/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-np-update-nodes-hcp-cluster-q4vvzb\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-q4vvzb/operationStatuses/08584279343875914687\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-vtr5fg/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:73]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0013642e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-update-nodes-hcp-cluster-q4vvzb\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-q4vvzb/operationStatuses/08584279343875914687\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-vtr5fg/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-214/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-np-update-nodes-hcp-cluster-q4vvzb\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-q4vvzb/operationStatuses/08584279343875914687\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-vtr5fg/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"a8288e4e9d33c7bec8313924a66a53288e81f74ee19c0f3f5dd0b4c7c8f87ad4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508825784389632","occurred_at":"2026-03-16T11:40:59Z","failed_at":"e2e","signature_id":"e16b386f7ec031654295cb32acdb3e87c443e348ab4fc85aa6e7149cc35b24c2","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000b54468\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-t7sblh\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a25a00\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-t7sblh\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a259a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-t7sblh\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a25980\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-t7sblh\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000b54468\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-t7sblh\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a25a00\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-t7sblh\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a259a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-t7sblh\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a25980\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-t7sblh\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"bd77d26b7b9cb11ad4b5c9cb45f2a2b77c33114ab771750d684a48a484a44013","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508825784389632","occurred_at":"2026-03-16T11:40:59Z","failed_at":"e2e","signature_id":"d2d47e8c674041b71cdc5c55a98af58f1624ee66cbce99f12d1337ee7edb9cf6","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000d14fa0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-19-8s2zrs-pvt4lx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-19-8s2zrs-pvt4lx/operationStatuses/08584279343979799486\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-cm6xfn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ihxzyyx7uepx6\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-278/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-19-8s2zrs-pvt4lx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-19-8s2zrs-pvt4lx/operationStatuses/08584279343979799486\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-cm6xfn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ihxzyyx7uepx6\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000d14fa0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-19-8s2zrs-pvt4lx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-19-8s2zrs-pvt4lx/operationStatuses/08584279343979799486\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-cm6xfn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ihxzyyx7uepx6\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-278/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-19-8s2zrs-pvt4lx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-19-8s2zrs-pvt4lx/operationStatuses/08584279343979799486\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-cm6xfn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ihxzyyx7uepx6\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"c3ac02c160da1f339e0112c10fafa7c99c9eb447a16630105c2e37c72c0b40b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508825784389632","occurred_at":"2026-03-16T11:40:59Z","failed_at":"e2e","signature_id":"ac0c022063fe3b30124b08268482555cac091965107029e25c4156ba18f0e43b","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001026570\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-52cdf9\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00088bd40\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-52cdf9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00088bd00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-52cdf9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00088bce0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-52cdf9\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001026570\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-52cdf9\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00088bd40\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-52cdf9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00088bd00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-52cdf9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00088bce0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-52cdf9\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"e4fefbec9e129744c20d3be8290cc53321c88266e9bd9a8536db2d6ab07b7ce0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508825784389632","occurred_at":"2026-03-16T11:40:59Z","failed_at":"e2e","signature_id":"70efe6b3a04cdd87fb4e70c35e45cf346a1d8c2d0bc0394b5a87f7beaded7a92","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:158]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001621d00\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-node-limit-cluster-jcvklv\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-jcvklv/operationStatuses/08584279343571105429\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-x4bdxj/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-36/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-node-limit-cluster-jcvklv\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-jcvklv/operationStatuses/08584279343571105429\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-x4bdxj/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscri...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:158]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001621d00\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-node-limit-cluster-jcvklv\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-jcvklv/operationStatuses/08584279343571105429\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-x4bdxj/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-36/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-node-limit-cluster-jcvklv\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-jcvklv/operationStatuses/08584279343571105429\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-x4bdxj/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscri...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"e50838487fd0a95cb3282c979dd8da22c51ad4d07f9970915212562ef1cac3d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508825784389632","occurred_at":"2026-03-16T11:40:59Z","failed_at":"e2e","signature_id":"4143b5c8f93f19294a05845e2a6a6fa2a15049f566cf893bcc094524179a1ce6","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.007s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000c58c20\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.007s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000c58c20\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}},{"row_id":"fa4658cb3a335b12602cc83a760093d6d9b1a891c742da274c678f0d0bb375e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508825784389632","occurred_at":"2026-03-16T11:40:59Z","failed_at":"e2e","signature_id":"746d5f32dd15acedc905f29f5ca9398a8607d74a112b2eafed998a0e13fe7474","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001560258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-46br52\" in resourcegroup=\"rg-zstream-upgrade-4-20-bk6w4d\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000da12c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-46br52\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-bk6w4d\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000da12a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-46br52\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-bk6w4d\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000da1280\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-20-46br52\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-bk6w4d\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001560258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-46br52\" in resourcegroup=\"rg-zstream-upgrade-4-20-bk6w4d\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000da12c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-46br52\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-bk6w4d\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000da12a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-46br52\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-bk6w4d\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000da1280\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-20-46br52\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-bk6w4d\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":9,"failure_pattern_ids":["567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d"],"matched_occurrences":9,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4411/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508624952725504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4411,"pr_state":"closed","pr_sha":"cb8e965f14c29db9495e70d63e92c4eb834a1e2e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:40:11Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"85e8719e1abd67629964a5e75fca9075888185cb42fddfc0b45c34002909f8ae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4411/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508624952725504","occurred_at":"2026-03-16T11:40:11Z","failed_at":"provision","signature_id":"27843ae452c73ac3bffe4c082d65982b73c2005ec9c5942193bc95d2e721159f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-azure-plugin-image","test_suite":"step graph","failure_text":"time=2026-03-16T12:32:28.753Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-azure-plugin-image description=\"Step mirror-azure-plugin-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e velero.azurePlugin.registry \u003cnil\u003e}:{\u003cnil\u003e velero.azurePlugin.repository \u003cnil\u003e}@{\u003cnil\u003e velero.azurePlugin.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-16T12:32:43.265Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-azure-plugin-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org\\nFetch pull secret for source registry registry.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nFailed to log in to ACR arohcpsvcdev: ERROR: Error loading command module 'acs': deadlock detected by _ModuleLock('azure.core.pipeline.transport') at 140702637496304\\nERROR: Error loading command module 'acr': cannot import name 'SpanKind' from partially initialized module 'azure.core.tracing' (most likely due to a circular import) (/usr/lib64/az/lib/python3.12/site-packages/azure/core/tracing/__init__.py)\\nERROR: Error loading command module 'appservice': cannot import name 'PipelineClient' from 'azure.core' (/usr/lib64/az/lib/python3.12/site-packages/azure/core/__init__.py)\\nERROR: 'acr' is misspelled or not recognized by the system.\\n\\nExamples from AI knowledge base:\\nhttps://aka.ms/cli_ref\\nRead more about the command in reference docs\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"41cea61b940716547ec2265592f122e93352921768862338021f0fc077c6bca6","failure_pattern":"ERROR: 'acr' is misspelled or not recognized by the system.","search_query":"ERROR: 'acr' is misspelled or not recognized by the system."}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["41cea61b940716547ec2265592f122e93352921768862338021f0fc077c6bca6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508515925987328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4412,"pr_state":"closed","pr_sha":"af4e1e32ae39029ac8be68f550bc0688e57f93a8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:39:45Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"1bea6c58f479b81579fc9e359850800329b4b6ce496e7aa0ed82f9778b9904c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508515925987328","occurred_at":"2026-03-16T11:39:45Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4413/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508497911451648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4413,"pr_state":"open","pr_sha":"f259c28b335a1c1a8410d3e1b1d83dc2732da5d3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:39:38Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"8441e54eb43f9a477e3748f68fd16fdd2737b860ee7283ce24379bf50ef95f1f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4413/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508497911451648","occurred_at":"2026-03-16T11:39:38Z","failed_at":"e2e","signature_id":"a9587a2b81ecf8fd2b199e0a754fdb238ee4d2ea8d771a97c6ea014e2225785f","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.003s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001035370\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.003s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001035370\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}},{"row_id":"a0482fea722a8a737d8d00b781d8b95c4d7601596425b7636429b89f187f13d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4413/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508497911451648","occurred_at":"2026-03-16T11:39:38Z","failed_at":"e2e","signature_id":"bc12c6ab8b101820c8b913b4ca77414e213458ce26b88b978cffb4b1e5a71874","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129]: Unexpected error:\n \u003c*exported.ResponseError | 0xc0013a6c90\u003e: \n GET https://rp.j1451648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1a0dba70-b3ba-402b-b761-254d517d4678\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1a0dba70-b3ba-402b-b761-254d517d4678\",\n \"name\": \"1a0dba70-b3ba-402b-b761-254d517d4678\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T13:01:52.936605636Z\",\n \"endTime\": \"2026-03-16T13:03:20.191663759Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Request-Id\": [\n \"4f06b70e-ab59-4022-aa8d-b8b5d1af60b7\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 13:03:23 GMT\",\n ],\n \"Content-Length\": [\"1559\"],\n \"X-Envoy-Upstream-Service-Time\": [\"11\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00099a640\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 49, 97, 48, 100, 98, 97, 55, 48, 45, 98, 51, 98, 97, 45, 52, 48, 50, 98, 45, 98, 55, 54, 49, 45, 50, 53, 52, 100, 53, 49, 55, 100, 52, 54, 55, 56, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 49, 97, 48, 100, 98, 97, 55, 48, 45, 98, 51, 98, 97, 45, 52, 48, 50, 98, 45, 98, 55, 54, 49, 45, 50, 53, 52, 100, 53, 49, 55, 100, 52, 54, 55, 56, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 116, 117, 115, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 114, 116, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 54, 84, 49, 51, 58, 48, 49, 58, 53, 50, 46, 57, 51, 54, 54, 48, 53, 54, 51, 54, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 110, 100, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 54, 84, 49, 51, 58, 48, 51, 58, 50, 48, 46, 49, 57, 49, 54, 54, 51, 55, 53, 57, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 102, 97, 105, 108, 101, 100, 32, 116, 111, 32, 103, 101, 116, 32, 109, 97, 110, 97, 103, 101, 100, 32, 105, 100, 101, 110, 116, 105, 116, 121, 32, 39, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 97, 114, 111, 45, 104, 99, 112, 45, 116, 101, 115, 116, 45, 109, 115, 105, 45, 99, 111, 110, 116, 97, 105, 110, 101, 114, 115, 45, 100, 101, 118, 45, 51, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 77, 97, 110, 97, 103, 101, 100, 73, 100, 101, 110, 116, 105, 116, 121, 47, 117, 115, 101, 114, 65, 115, 115, 105, 103, 110, 101, 100, 73, 100, 101, 110, 116, 105, 116, 105, 101, 115, 47, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 39, 58, 32, 71, 69, 84, 32, 104, 116, 116, 112, 115, 58, 47, 47, 109, 97, 110, 97, 103, 101, 109, 101, 110, 116, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 11...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129]: Unexpected error:\n \u003c*exported.ResponseError | 0xc0013a6c90\u003e: \n GET https://rp.j1451648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1a0dba70-b3ba-402b-b761-254d517d4678\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1a0dba70-b3ba-402b-b761-254d517d4678\",\n \"name\": \"1a0dba70-b3ba-402b-b761-254d517d4678\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T13:01:52.936605636Z\",\n \"endTime\": \"2026-03-16T13:03:20.191663759Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Request-Id\": [\n \"4f06b70e-ab59-4022-aa8d-b8b5d1af60b7\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 13:03:23 GMT\",\n ],\n \"Content-Length\": [\"1559\"],\n \"X-Envoy-Upstream-Service-Time\": [\"11\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00099a640\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 49, 97, 48, 100, 98, 97, 55, 48, 45, 98, 51, 98, 97, 45, 52, 48, 50, 98, 45, 98, 55, 54, 49, 45, 50, 53, 52, 100, 53, 49, 55, 100, 52, 54, 55, 56, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 49, 97, 48, 100, 98, 97, 55, 48, 45, 98, 51, 98, 97, 45, 52, 48, 50, 98, 45, 98, 55, 54, 49, 45, 50, 53, 52, 100, 53, 49, 55, 100, 52, 54, 55, 56, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 116, 117, 115, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 114, 116, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 54, 84, 49, 51, 58, 48, 49, 58, 53, 50, 46, 57, 51, 54, 54, 48, 53, 54, 51, 54, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 110, 100, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 54, 84, 49, 51, 58, 48, 51, 58, 50, 48, 46, 49, 57, 49, 54, 54, 51, 55, 53, 57, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 102, 97, 105, 108, 101, 100, 32, 116, 111, 32, 103, 101, 116, 32, 109, 97, 110, 97, 103, 101, 100, 32, 105, 100, 101, 110, 116, 105, 116, 121, 32, 39, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 97, 114, 111, 45, 104, 99, 112, 45, 116, 101, 115, 116, 45, 109, 115, 105, 45, 99, 111, 110, 116, 97, 105, 110, 101, 114, 115, 45, 100, 101, 118, 45, 51, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 77, 97, 110, 97, 103, 101, 100, 73, 100, 101, 110, 116, 105, 116, 121, 47, 117, 115, 101, 114, 65, 115, 115, 105, 103, 110, 101, 100, 73, 100, 101, 110, 116, 105, 116, 105, 101, 115, 47, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 39, 58, 32, 71, 69, 84, 32, 104, 116, 116, 112, 115, 58, 47, 47, 109, 97, 110, 97, 103, 101, 109, 101, 110, 116, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 11...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c5e96bce7472d49951746ba430b6f10dd49f64854a93287a5feb7ca9e953565c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4413/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508497911451648","occurred_at":"2026-03-16T11:39:38Z","failed_at":"e2e","signature_id":"9fd3c310d35f8d5877a3b749a6ea533382a732785e16d06d714a0886e8c1337a","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:86]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000492a60\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-6zfqcn\" to finish creating: GET https://rp.j1451648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f143b52b-7509-495c-a365-0983468be9fc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f143b52b-7509-495c-a365-0983468be9fc\",\n \"name\": \"f143b52b-7509-495c-a365-0983468be9fc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T13:06:15.755983798Z\",\n \"endTime\": \"2026-03-16T13:07:20.629561299Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-6zfqcn\\\" to finish creating: GET https://rp.j1451648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f143b52b-7509-495c-a365-0983468be9fc\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f143b52b-7509-495c-a365-0983468be9fc\\\",\\n \\\"name\\\": \\\"f143b52b-7509-495c-a365-0983468be9fc\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T13:06:15.755983798Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T13:07:20.629561299Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000492a40\u003e{\n msg: \"failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-6zfqcn\\\" to finish creating: GET https://rp.j1451648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f143b52b-7509-495c-a365-0983468be9fc\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f143b52b-7509-495c-a365-0983468be9fc\\\",\\n \\\"name\\\": \\\"f143b52b-7509-495c-a365-0983468be9fc\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T13:06:15.755983798Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T13:07:20.629561299Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n---------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:86]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000492a60\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-6zfqcn\" to finish creating: GET https://rp.j1451648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f143b52b-7509-495c-a365-0983468be9fc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f143b52b-7509-495c-a365-0983468be9fc\",\n \"name\": \"f143b52b-7509-495c-a365-0983468be9fc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T13:06:15.755983798Z\",\n \"endTime\": \"2026-03-16T13:07:20.629561299Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-6zfqcn\\\" to finish creating: GET https://rp.j1451648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f143b52b-7509-495c-a365-0983468be9fc\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f143b52b-7509-495c-a365-0983468be9fc\\\",\\n \\\"name\\\": \\\"f143b52b-7509-495c-a365-0983468be9fc\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T13:06:15.755983798Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T13:07:20.629561299Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000492a40\u003e{\n msg: \"failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-6zfqcn\\\" to finish creating: GET https://rp.j1451648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f143b52b-7509-495c-a365-0983468be9fc\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f143b52b-7509-495c-a365-0983468be9fc\\\",\\n \\\"name\\\": \\\"f143b52b-7509-495c-a365-0983468be9fc\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T13:06:15.755983798Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T13:07:20.629561299Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n---------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"de36a540ce0eeee4e4e5cbb7cc4144df93b1f2bb9f3d9355da7a84ea5779394c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4413/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508497911451648","occurred_at":"2026-03-16T11:39:38Z","failed_at":"e2e","signature_id":"200fab97419ee03852608770a7bf669f15b0a6eab1288206cb377ee16bb33e77","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:390]: Timed out after 600.002s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:389 with:\nAll ClusterOperators should report Available=True, but these are not available: [image-registry (Available=False)]\nExpected\n \u003c[]string | len:1, cap:1\u003e: [\n \"image-registry (Available=False)\",\n ]\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:390]: Timed out after 600.002s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:389 with:\nAll ClusterOperators should report Available=True, but these are not available: [image-registry (Available=False)]\nExpected\n \u003c[]string | len:1, cap:1\u003e: [\n \"image-registry (Available=False)\",\n ]\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d24e138f3b84b77b0ea6793407ac1f41343d0658f0c69312eac159e4f4f7eb45","failure_pattern":"All ClusterOperators should report Available=True, but these are not available: [image-registry (Available=False)]","search_query":"All ClusterOperators should report Available=True, but these are not available: [image-registry (Available=False)]"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","d24e138f3b84b77b0ea6793407ac1f41343d0658f0c69312eac159e4f4f7eb45"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508491213148160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4414,"pr_state":"closed","pr_sha":"2b8f05f7b9ca619151265287d1bc0e81026e78c6","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:39:31Z"},"failed_at":["e2e"],"failed_test_count":16,"occurrences":[{"row_id":"133bd5a468e40e35c9d2f8f8670bcfdb572120a3acb7bbd2700cc440b15e78d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508491213148160","occurred_at":"2026-03-16T11:39:31Z","failed_at":"e2e","signature_id":"4ddeec9c7b7b6482f6de4b0a75f499a2ac2f34cbe449ba23219600295a41bf74","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0015221e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listjbszpx\" in resourcegroup=\"ea-list-ft7pjx\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001139140\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listjbszpx\\\" in resourcegroup=\\\"ea-list-ft7pjx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001139120\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listjbszpx\\\" in resourcegroup=\\\"ea-list-ft7pjx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001139100\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listjbszpx\\\" in resourcegroup=\\\"ea-list-ft7pjx\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0990c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0015221e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listjbszpx\" in resourcegroup=\"ea-list-ft7pjx\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001139140\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listjbszpx\\\" in resourcegroup=\\\"ea-list-ft7pjx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001139120\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listjbszpx\\\" in resourcegroup=\\\"ea-list-ft7pjx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001139100\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listjbszpx\\\" in resourcegroup=\\\"ea-list-ft7pjx\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0990c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"4177054afceed7f52e0751340dc1408f0a050fd26caa7843693c6fb3b24db97a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508491213148160","occurred_at":"2026-03-16T11:39:31Z","failed_at":"e2e","signature_id":"a1f569ff5d175edae50257eeba3c0b837cb7ac9b6c62b027aa1ff450198587d0","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0014f61c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-c97b6s\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000ccd7a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-c97b6s\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ccd780\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-c97b6s\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ccd760\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-c97b6s\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0990c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0014f61c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-c97b6s\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000ccd7a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-c97b6s\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ccd780\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-c97b6s\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ccd760\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-c97b6s\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0990c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"477de6e55e9052b8df4b77897db77c467ef42aa6fed3268ecf78d012b2e798b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508491213148160","occurred_at":"2026-03-16T11:39:31Z","failed_at":"e2e","signature_id":"aec731b8a14d42b71fb35717c3cbb9757aee0ce6fa76ac4ecbef467eee44a90e","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000190fa8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-hkbztn\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000981d60\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-hkbztn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000981d40\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-hkbztn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000981d20\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-hkbztn\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0990c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000190fa8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-hkbztn\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000981d60\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-hkbztn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000981d40\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-hkbztn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000981d20\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-hkbztn\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0990c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"491c81a5795eae187e29c31a8dce05f4bf552c49bcc8829157813d696e430e6f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508491213148160","occurred_at":"2026-03-16T11:39:31Z","failed_at":"e2e","signature_id":"d970cc4765f912f21ccf1b74fb328c6def09a426e0e00d3bca93f4e62a6321f4","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:242]: Unexpected error:\n \u003c*errors.joinError | 0xc0010301b0\u003e: \n context deadline exceeded\n failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\n {\n errs: [\n \u003ccontext.deadlineExceededError\u003e{},\n \u003c*fmt.wrapError | 0xc000797340\u003e{\n msg: \"failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xc000797300\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:242]: Unexpected error:\n \u003c*errors.joinError | 0xc0010301b0\u003e: \n context deadline exceeded\n failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\n {\n errs: [\n \u003ccontext.deadlineExceededError\u003e{},\n \u003c*fmt.wrapError | 0xc000797340\u003e{\n msg: \"failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xc000797300\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"519185a1b11a2e8e771e7056eb4adcf297f7e3315b61a48406edad0aaf706efe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508491213148160","occurred_at":"2026-03-16T11:39:31Z","failed_at":"e2e","signature_id":"b3cfe62df2e40e5c95f94ff64ee4081322d05f6d051f62d468297eee893ec59b","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.006s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000fa96a0\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.006s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000fa96a0\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}},{"row_id":"581a02f42ad1ba6af9e6380a60f81b2fc2c5dfafae758c4dbca1c27cb4e38b12","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508491213148160","occurred_at":"2026-03-16T11:39:31Z","failed_at":"e2e","signature_id":"c6e23cf5cf621b41808c1bb1a263977181349f229ed28c3def400fe6f5abc206","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000a24780\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-w6lldn\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000979be0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-w6lldn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000979bc0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-w6lldn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000979b60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-w6lldn\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0990c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000a24780\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-w6lldn\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000979be0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-w6lldn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000979bc0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-w6lldn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000979b60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-w6lldn\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0990c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"5c3edd37e9e2b429f30ce2a71af33985ec7aed4cd13e3fd5940454b27978cd13","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508491213148160","occurred_at":"2026-03-16T11:39:31Z","failed_at":"e2e","signature_id":"1f892eda52555e5cbba32433bced2aad76b24f7e946306e58838bfd5a58f3852","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000413360\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-sdxgfv\" to finish creating: GET https://rp.j3148160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6883ff96-373a-49d2-8850-c1af1afe4828\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6883ff96-373a-49d2-8850-c1af1afe4828\",\n \"name\": \"6883ff96-373a-49d2-8850-c1af1afe4828\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:36:35.110062481Z\",\n \"endTime\": \"2026-03-16T14:37:50.877973227Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-75/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-75/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-sdxgfv\\\" to finish creating: GET https://rp.j3148160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6883ff96-373a-49d2-8850-c1af1afe4828\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6883ff96-373a-49d2-8850-c1af1afe4828\\\",\\n \\\"name\\\": \\\"6883ff96-373a-49d2-8850-c1af1afe4828\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:36:35.110062481Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:37:50.877973227Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-75/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-75/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000413320\u003e{\n msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-sdxgfv\\\" to finish creating: GET https://rp.j3148160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6883ff96-373a-49d2-8850-c1af1afe4828\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6883ff96-373a-49d2-8850-c1af1afe4828\\\",\\n \\\"name\\\": \\\"6883ff96-373a-49d2-8850-c1af1afe4828\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:36:35.110062481Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:37:50.877973227Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-75/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-75/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n----------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000413360\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-sdxgfv\" to finish creating: GET https://rp.j3148160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6883ff96-373a-49d2-8850-c1af1afe4828\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6883ff96-373a-49d2-8850-c1af1afe4828\",\n \"name\": \"6883ff96-373a-49d2-8850-c1af1afe4828\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:36:35.110062481Z\",\n \"endTime\": \"2026-03-16T14:37:50.877973227Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-75/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-75/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-sdxgfv\\\" to finish creating: GET https://rp.j3148160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6883ff96-373a-49d2-8850-c1af1afe4828\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6883ff96-373a-49d2-8850-c1af1afe4828\\\",\\n \\\"name\\\": \\\"6883ff96-373a-49d2-8850-c1af1afe4828\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:36:35.110062481Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:37:50.877973227Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-75/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-75/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000413320\u003e{\n msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-sdxgfv\\\" to finish creating: GET https://rp.j3148160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6883ff96-373a-49d2-8850-c1af1afe4828\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6883ff96-373a-49d2-8850-c1af1afe4828\\\",\\n \\\"name\\\": \\\"6883ff96-373a-49d2-8850-c1af1afe4828\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:36:35.110062481Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:37:50.877973227Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-75/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-75/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n----------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"624ce7743b9d8f2f13f0ec1203f882eb48f1bddab357df674ae9f0d7dbceecac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508491213148160","occurred_at":"2026-03-16T11:39:31Z","failed_at":"e2e","signature_id":"f91073499b2fe9701cb0ab0acb72644cbc2f4331c7dcb48214d46576a99aa762","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:73]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00150c520\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-update-nodes-hcp-cluster-pqknpk\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-pqknpk/operationStatuses/08584279348255055112\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-v6hvbv/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-75/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-np-update-nodes-hcp-cluster-pqknpk\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-pqknpk/operationStatuses/08584279348255055112\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-v6hvbv/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:73]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00150c520\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-update-nodes-hcp-cluster-pqknpk\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-pqknpk/operationStatuses/08584279348255055112\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-v6hvbv/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-75/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-np-update-nodes-hcp-cluster-pqknpk\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-pqknpk/operationStatuses/08584279348255055112\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-v6hvbv/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"6bb48e2242fd9ebaaaf7d5b047abdaad566e74601f1a6f89e7ec607463bd0ddc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508491213148160","occurred_at":"2026-03-16T11:39:31Z","failed_at":"e2e","signature_id":"bdaae6176d3268a8f211d3e818e2ba6b5e2d7cbc916058ba5064e2ca964625a1","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000a40498\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-fl5mzp\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0013d7200\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-fl5mzp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0013d71e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-fl5mzp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0013d71c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-fl5mzp\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0990c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000a40498\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-fl5mzp\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0013d7200\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-fl5mzp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0013d71e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-fl5mzp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0013d71c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-fl5mzp\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0990c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"83a932644b7ec11b2c79f0ebf8c322a53894b038c04e95b43d0a430025d67bdc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508491213148160","occurred_at":"2026-03-16T11:39:31Z","failed_at":"e2e","signature_id":"31e75017c0f747a9c3483474caaa1bdb1e95b9268ac3754ef4d27a6d36d90fc8","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00049fa40\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-20-n6hv7n-4x29z4\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-n6hv7n-4x29z4/operationStatuses/08584279346342706585\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-75/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-20-n6hv7n-4x29z4\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-n6hv7n-4x29z4/operationStatuses/08584279346342706585\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-75/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00049fa20\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-20-n6hv7n-4x29z4\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-n6hv7n-4x29z4/operationStatuses/08584279346342706585\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-75/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list depl...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00049fa40\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-20-n6hv7n-4x29z4\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-n6hv7n-4x29z4/operationStatuses/08584279346342706585\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-75/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-20-n6hv7n-4x29z4\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-n6hv7n-4x29z4/operationStatuses/08584279346342706585\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-75/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00049fa20\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-20-n6hv7n-4x29z4\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-n6hv7n-4x29z4/operationStatuses/08584279346342706585\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-75/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list depl...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"8f7a257190093c623e21d70063dac88dd08f684ad97fa9afd63b91f3fdd72b52","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508491213148160","occurred_at":"2026-03-16T11:39:31Z","failed_at":"e2e","signature_id":"d97ddf9469d5035c504fbbd00ec17927016c84ab6bef7cc1309e41d5c13a80e4","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0014c0360\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-79p6sp\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000e4a6a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-79p6sp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e4a680\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-79p6sp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e4a660\u003e{\n msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-79p6sp\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0990c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0014c0360\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-79p6sp\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000e4a6a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-79p6sp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e4a680\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-79p6sp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e4a660\u003e{\n msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-79p6sp\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0990c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"93def0547f436504616bbd6ac5f7acb0f325c01cea7b52a1261e47a67564eb89","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508491213148160","occurred_at":"2026-03-16T11:39:31Z","failed_at":"e2e","signature_id":"c18aadf010bd54518f33bd42f9593b72c81d48329382b51770932abd95be0558","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:86]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0016254e0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-h89ld2\" to finish creating: GET https://rp.j3148160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ebe17b9b-b660-46f1-be75-e057609d910e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ebe17b9b-b660-46f1-be75-e057609d910e\",\n \"name\": \"ebe17b9b-b660-46f1-be75-e057609d910e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:47:24.104802433Z\",\n \"endTime\": \"2026-03-16T14:48:51.18681009Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Failed to get virtual network 'customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-h89ld2/providers/Microsoft.Network/virtualNetworks/customer-vnet\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-h89ld2\\\" to finish creating: GET https://rp.j3148160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ebe17b9b-b660-46f1-be75-e057609d910e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ebe17b9b-b660-46f1-be75-e057609d910e\\\",\\n \\\"name\\\": \\\"ebe17b9b-b660-46f1-be75-e057609d910e\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:47:24.104802433Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:48:51.18681009Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-h89ld2/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0016254c0\u003e{\n msg: \"failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-h89ld2\\\" to finish creating: GET https://rp.j3148160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ebe17b9b-b660-46f1-be75-e057609d910e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ebe17b9b-b660-46f1-be75-e057609d910e\\\",\\n \\\"name\\\": \\\"ebe17b9b-b660-46f1-be75-e057609d910e\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:47:24.104802433Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:48:51.18681009Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-h89ld2/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:86]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0016254e0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-h89ld2\" to finish creating: GET https://rp.j3148160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ebe17b9b-b660-46f1-be75-e057609d910e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ebe17b9b-b660-46f1-be75-e057609d910e\",\n \"name\": \"ebe17b9b-b660-46f1-be75-e057609d910e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:47:24.104802433Z\",\n \"endTime\": \"2026-03-16T14:48:51.18681009Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Failed to get virtual network 'customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-h89ld2/providers/Microsoft.Network/virtualNetworks/customer-vnet\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-h89ld2\\\" to finish creating: GET https://rp.j3148160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ebe17b9b-b660-46f1-be75-e057609d910e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ebe17b9b-b660-46f1-be75-e057609d910e\\\",\\n \\\"name\\\": \\\"ebe17b9b-b660-46f1-be75-e057609d910e\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:47:24.104802433Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:48:51.18681009Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-h89ld2/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0016254c0\u003e{\n msg: \"failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-h89ld2\\\" to finish creating: GET https://rp.j3148160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ebe17b9b-b660-46f1-be75-e057609d910e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ebe17b9b-b660-46f1-be75-e057609d910e\\\",\\n \\\"name\\\": \\\"ebe17b9b-b660-46f1-be75-e057609d910e\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:47:24.104802433Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:48:51.18681009Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-h89ld2/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b8360754eb30611f6bed1c32e4b7ca58006c4903d818929029e6afed6f3b95a1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508491213148160","occurred_at":"2026-03-16T11:39:31Z","failed_at":"e2e","signature_id":"3e50b3b6acafe713a28a07175a807274f9126aac149962c3dc1eb2f09270f74a","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001804360\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-9b65vd\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a1c580\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-9b65vd\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a1c520\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-9b65vd\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a1c480\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-9b65vd\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0990c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001804360\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-9b65vd\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a1c580\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-9b65vd\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a1c520\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-9b65vd\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a1c480\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-9b65vd\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0990c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"ce4719267f75e8e992b6e8ee19d5d2608b2b11524d5fcde98a703a8fca3093ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508491213148160","occurred_at":"2026-03-16T11:39:31Z","failed_at":"e2e","signature_id":"e906d89b0e565dd30f0672be7b5ae272f0a684713332129c7adedd394fb03283","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ff01e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-x7q5bp\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f10de0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-x7q5bp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f10dc0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-x7q5bp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f10da0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-x7q5bp\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0990c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ff01e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-x7q5bp\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f10de0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-x7q5bp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f10dc0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-x7q5bp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f10da0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-x7q5bp\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0990c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"e7a0c7b9d3bdb1ed79367af4319643f9adfa740bc41443c63f4baba554e5b774","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508491213148160","occurred_at":"2026-03-16T11:39:31Z","failed_at":"e2e","signature_id":"3d213e4c58e130a61f9911c2933b88d64531c2cf6f3a4b455c49e5bff66df797","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ee6228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-5qv9g7\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0014a6f40\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-5qv9g7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0014a6f20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-5qv9g7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0014a6f00\u003e{\n msg: \"failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-5qv9g7\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0990c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ee6228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-5qv9g7\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0014a6f40\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-5qv9g7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0014a6f20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-5qv9g7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0014a6f00\u003e{\n msg: \"failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-5qv9g7\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0990c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"f1140cd7996d76c84a8c219169b9362778497285b97391bfdd59ca5ba6e74933","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508491213148160","occurred_at":"2026-03-16T11:39:31Z","failed_at":"e2e","signature_id":"d5c2bab4b9568a99818b4e90d37676580f032d4418acdae7c14b6e8e84d7833e","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e84198\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-kqqw8g\" in resourcegroup=\"rg-zstream-upgrade-4-21-t9b2n6\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001271080\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-kqqw8g\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-t9b2n6\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001271060\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-kqqw8g\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-t9b2n6\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001271040\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-kqqw8g\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-t9b2n6\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0990c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e84198\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-kqqw8g\" in resourcegroup=\"rg-zstream-upgrade-4-21-t9b2n6\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001271080\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-kqqw8g\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-t9b2n6\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001271060\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-kqqw8g\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-t9b2n6\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001271040\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-kqqw8g\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-t9b2n6\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0990c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":16,"failure_pattern_ids":["503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d"],"matched_occurrences":16,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4425/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508445902082048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4425,"pr_state":"closed","pr_sha":"2d42e9872e1b844e7081e8d59c4459d9a907f5ab","final_merged_sha":"855eea85f3052237fc8528c6d280c0ac41ee4307","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:39:24Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ccb6cdb5e18019970f565f5a569de82799ac1fb23565d78ad62ec5d1a46507e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4425/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508445902082048","occurred_at":"2026-03-16T11:39:24Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508362448015360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4433,"pr_state":"closed","pr_sha":"599dd2c53ef9a79607254436be4a2ed8e0671e8c","final_merged_sha":"599dd2c53ef9a79607254436be4a2ed8e0671e8c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T11:39:08Z"},"failed_at":["e2e"],"failed_test_count":16,"occurrences":[{"row_id":"149db3ec1878d13b8a400a660732def085285d14ab735a1d1b14fc4e39fe0092","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508362448015360","occurred_at":"2026-03-16T11:39:08Z","failed_at":"e2e","signature_id":"d16b4ef51a6b4b673c4fd336229b7c47534983be72a4c27f68c189b6aa8c4735","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:390]: Timed out after 600.001s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:389 with:\nAll ClusterOperators should report Available=True, but these are not available: [image-registry (Available=False)]\nExpected\n \u003c[]string | len:1, cap:1\u003e: [\n \"image-registry (Available=False)\",\n ]\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:390]: Timed out after 600.001s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:389 with:\nAll ClusterOperators should report Available=True, but these are not available: [image-registry (Available=False)]\nExpected\n \u003c[]string | len:1, cap:1\u003e: [\n \"image-registry (Available=False)\",\n ]\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d24e138f3b84b77b0ea6793407ac1f41343d0658f0c69312eac159e4f4f7eb45","failure_pattern":"All ClusterOperators should report Available=True, but these are not available: [image-registry (Available=False)]","search_query":"All ClusterOperators should report Available=True, but these are not available: [image-registry (Available=False)]"}},{"row_id":"1580c7187c8dab5e4831b8921d4efb216918776aa322ffe8766cd92ede6eadc4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508362448015360","occurred_at":"2026-03-16T11:39:08Z","failed_at":"e2e","signature_id":"1e6b0301ac5725d0cd383811b773ccf9e77b7bff28520860b0d4b117a888f61f","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0017b2240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-hblwz4\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0017893e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-hblwz4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0017893c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-hblwz4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001789380\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-hblwz4\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0017b2240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-hblwz4\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0017893e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-hblwz4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0017893c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-hblwz4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001789380\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-hblwz4\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"4ac12c7e2c21e4b5ae57894ebdf8c56472027b112faa4ac797f32376dfd5276b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508362448015360","occurred_at":"2026-03-16T11:39:08Z","failed_at":"e2e","signature_id":"4f36e225eb3c2374f82576fdb4c57bca05d2290ac4323fd5001827c74a3b4b88","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0008d6580\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-f6npnk\" to finish creating: GET https://rp.j8015360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e1fe82d-4e07-4f03-a07b-d87cb2e29dd4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e1fe82d-4e07-4f03-a07b-d87cb2e29dd4\",\n \"name\": \"5e1fe82d-4e07-4f03-a07b-d87cb2e29dd4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:51:40.269113062Z\",\n \"endTime\": \"2026-03-16T14:53:13.013429678Z\",\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"Cluster provisioning failed due to multiple errors\",\n \"details\": [\n {\n \"code\": \"InternalServerError\",\n \"message\": \"Failed to get virtual network 'customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-f6npnk/providers/Microsoft.Network/virtualNetworks/customer-vnet\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n },\n {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-f6npnk\\\" to finish creating: GET https://rp.j8015360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e1fe82d-4e07-4f03-a07b-d87cb2e29dd4\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e1fe82d-4e07-4f03-a07b-d87cb2e29dd4\\\",\\n \\\"name\\\": \\\"5e1fe82d-4e07-4f03-a07b-d87cb2e29dd4\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:51:40.269113062Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:53:13.013429678Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-f6npnk/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n },\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0008d6540\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-f6npnk\\\" to finish creating: GET https://rp.j8015360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e1fe82d-4e07-4f03-a07b-d87cb2e29dd4\\n---------------------------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0008d6580\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-f6npnk\" to finish creating: GET https://rp.j8015360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e1fe82d-4e07-4f03-a07b-d87cb2e29dd4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e1fe82d-4e07-4f03-a07b-d87cb2e29dd4\",\n \"name\": \"5e1fe82d-4e07-4f03-a07b-d87cb2e29dd4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:51:40.269113062Z\",\n \"endTime\": \"2026-03-16T14:53:13.013429678Z\",\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"Cluster provisioning failed due to multiple errors\",\n \"details\": [\n {\n \"code\": \"InternalServerError\",\n \"message\": \"Failed to get virtual network 'customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-f6npnk/providers/Microsoft.Network/virtualNetworks/customer-vnet\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n },\n {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-f6npnk\\\" to finish creating: GET https://rp.j8015360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e1fe82d-4e07-4f03-a07b-d87cb2e29dd4\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e1fe82d-4e07-4f03-a07b-d87cb2e29dd4\\\",\\n \\\"name\\\": \\\"5e1fe82d-4e07-4f03-a07b-d87cb2e29dd4\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:51:40.269113062Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:53:13.013429678Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-f6npnk/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n },\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0008d6540\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-f6npnk\\\" to finish creating: GET https://rp.j8015360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e1fe82d-4e07-4f03-a07b-d87cb2e29dd4\\n---------------------------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0aa9ed5d94e7167a12aa30282eff19539fc3fc473772cb96b223d420df66775f","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"8133a3676c406b42128572c6f4d45c047931bd9438ed7073f10c5cc8a27f62ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508362448015360","occurred_at":"2026-03-16T11:39:08Z","failed_at":"e2e","signature_id":"b2150096938fe6e81966d5da1395b46d7e1e7c8ffebe2ed250d4872c3ae96f81","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001640168\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-nz84xw\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00135ee40\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-nz84xw\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00135ee20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-nz84xw\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00135ee00\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-nz84xw\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001640168\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-nz84xw\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00135ee40\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-nz84xw\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00135ee20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-nz84xw\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00135ee00\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-nz84xw\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"a10e4cdd15f5d7d0f70b2316de80bab57ce2bcd65852795314a4b9e322105c83","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508362448015360","occurred_at":"2026-03-16T11:39:08Z","failed_at":"e2e","signature_id":"1c0b2f954200b6e2ce88b210181b986160a671278f46011ba9723710e596b791","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0011baba0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-20-9lh2mn-nbskpm\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-9lh2mn-nbskpm/operationStatuses/08584279343902214935\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-x6l9xl/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-rj3ukhr2ls7pg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-37/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-20-9lh2mn-nbskpm\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-9lh2mn-nbskpm/operationStatuses/08584279343902214935\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-x6l9xl/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-rj3ukhr2ls7pg\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0011baba0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-20-9lh2mn-nbskpm\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-9lh2mn-nbskpm/operationStatuses/08584279343902214935\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-x6l9xl/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-rj3ukhr2ls7pg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-37/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-20-9lh2mn-nbskpm\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-9lh2mn-nbskpm/operationStatuses/08584279343902214935\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-x6l9xl/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-rj3ukhr2ls7pg\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"a13f61e7a49aa5ebdf930877b7405003fc207edc93d940aeb7b555a679304725","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508362448015360","occurred_at":"2026-03-16T11:39:08Z","failed_at":"e2e","signature_id":"f815cb34e9ef8a7f3a4243ff9ad1ae56ec51faeacf222c5d835103d4890530e8","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f58330\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-gccfmc\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0009319a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-gccfmc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000931980\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-gccfmc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000931960\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-gccfmc\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f58330\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-gccfmc\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0009319a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-gccfmc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000931980\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-gccfmc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000931960\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-gccfmc\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"a7c858f8c3dfee987c6350d22c0c82c3e4f68a24d6cf095f14ad0e24a5090e6f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508362448015360","occurred_at":"2026-03-16T11:39:08Z","failed_at":"e2e","signature_id":"61d241560d41eca56a2848f9a60d4d6226615538e2d9ba33366c81889d349d3c","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:151]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0013f01e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster3-gck6vl\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster3-gck6vl/operationStatuses/08584279347703054161\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-qkqcp2/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-igpwm4vimfr3y\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-basic-hcp-cluster3-gck6vl\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster3-gck6vl/operationStatuses/08584279347703054161\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-qkqcp2/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-igpwm4vimfr3y\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0013f01c0\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-basic-hcp-cluster3-gck6vl\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster3-gck6vl/operationStatuses/085842793477...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:151]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0013f01e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster3-gck6vl\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster3-gck6vl/operationStatuses/08584279347703054161\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-qkqcp2/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-igpwm4vimfr3y\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-basic-hcp-cluster3-gck6vl\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster3-gck6vl/operationStatuses/08584279347703054161\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-qkqcp2/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-igpwm4vimfr3y\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0013f01c0\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-basic-hcp-cluster3-gck6vl\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster3-gck6vl/operationStatuses/085842793477...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"aee5a2ac213a3bd6d42771d3524f4ae1fee2a512b34eaa4f44b66d55bd6e2f23","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508362448015360","occurred_at":"2026-03-16T11:39:08Z","failed_at":"e2e","signature_id":"53f580932f4ae7227a50b4ef0c7fe8abc9459faae66214a6b992fdfab278d22f","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001624198\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-z64tpk\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00097f180\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-z64tpk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00097f160\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-z64tpk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00097f140\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-z64tpk\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001624198\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-z64tpk\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00097f180\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-z64tpk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00097f160\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-z64tpk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00097f140\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-z64tpk\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"b1ec74cdd772dd58f93ceaf2ea4411902b152290104790a18184613b5b387a2f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508362448015360","occurred_at":"2026-03-16T11:39:08Z","failed_at":"e2e","signature_id":"aecb180dbb1d3d497eec0e9e6f64f5cbc4cd599089904961596c68e5c4f3fbca","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000868480\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-hjjnp8\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000e44fe0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-hjjnp8\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e44fc0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-hjjnp8\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e44fa0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-hjjnp8\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000868480\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-hjjnp8\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000e44fe0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-hjjnp8\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e44fc0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-hjjnp8\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e44fa0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-hjjnp8\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"bcbac6954f69c7428516b204b2e14edcaaa3bca96e0d3b29ed7ab572262aca41","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508362448015360","occurred_at":"2026-03-16T11:39:08Z","failed_at":"e2e","signature_id":"389acb5489721766d2c2421cdb4b6f881fc820e64371feadc7699de4be956de4","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e2e240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-8rxw7g\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001341760\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-8rxw7g\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001341740\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-8rxw7g\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001341720\u003e{\n msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-8rxw7g\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e2e240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-8rxw7g\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001341760\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-8rxw7g\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001341740\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-8rxw7g\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001341720\u003e{\n msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-8rxw7g\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"c249cc6600cedd570ad5a81857cfdb0605bc030a9cffa46d1f95cfa57f2efbb3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508362448015360","occurred_at":"2026-03-16T11:39:08Z","failed_at":"e2e","signature_id":"aa120139eb3b99cc668a60846ad281cdcb9bf27358fb6150615bceaa51e8d10c","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000a345e8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-22j64g\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000e0d260\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-22j64g\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e0d240\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-22j64g\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e0d220\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-22j64g\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000a345e8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-22j64g\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000e0d260\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-22j64g\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e0d240\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-22j64g\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e0d220\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-22j64g\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"c6c402606c725f5bef63636c22cf222a80c4698a51dc485447ff964fb850fa1c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508362448015360","occurred_at":"2026-03-16T11:39:08Z","failed_at":"e2e","signature_id":"92a14be066c36cdcb0710bbcc02e8f05b191b3fa1aa5400bbfe0a863c838058a","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001621880\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-rwkvnl\" to finish creating: GET https://rp.j8015360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b71f3c20-bebb-45a9-8de6-fc37671e60f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b71f3c20-bebb-45a9-8de6-fc37671e60f5\",\n \"name\": \"b71f3c20-bebb-45a9-8de6-fc37671e60f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:55:54.425365996Z\",\n \"endTime\": \"2026-03-16T14:56:39.660041286Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-253/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-253/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-rwkvnl\\\" to finish creating: GET https://rp.j8015360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b71f3c20-bebb-45a9-8de6-fc37671e60f5\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b71f3c20-bebb-45a9-8de6-fc37671e60f5\\\",\\n \\\"name\\\": \\\"b71f3c20-bebb-45a9-8de6-fc37671e60f5\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:55:54.425365996Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:56:39.660041286Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-253/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-253/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001621860\u003e{\n msg: \"failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-rwkvnl\\\" to finish creating: GET https://rp.j8015360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b71f3c20-bebb-45a9-8de6-fc37671e60f5\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b71f3c20-bebb-45a9-8de6-fc37671e60f5\\\",\\n \\\"name\\\": \\\"b71f3c20-bebb-45a9-8de6-fc37671e60f5\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:55:54.425365996Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:56:39.660041286Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-253/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-253/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: Subscript...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001621880\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-rwkvnl\" to finish creating: GET https://rp.j8015360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b71f3c20-bebb-45a9-8de6-fc37671e60f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b71f3c20-bebb-45a9-8de6-fc37671e60f5\",\n \"name\": \"b71f3c20-bebb-45a9-8de6-fc37671e60f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:55:54.425365996Z\",\n \"endTime\": \"2026-03-16T14:56:39.660041286Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-253/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-253/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-rwkvnl\\\" to finish creating: GET https://rp.j8015360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b71f3c20-bebb-45a9-8de6-fc37671e60f5\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b71f3c20-bebb-45a9-8de6-fc37671e60f5\\\",\\n \\\"name\\\": \\\"b71f3c20-bebb-45a9-8de6-fc37671e60f5\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:55:54.425365996Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:56:39.660041286Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-253/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-253/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001621860\u003e{\n msg: \"failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-rwkvnl\\\" to finish creating: GET https://rp.j8015360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b71f3c20-bebb-45a9-8de6-fc37671e60f5\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b71f3c20-bebb-45a9-8de6-fc37671e60f5\\\",\\n \\\"name\\\": \\\"b71f3c20-bebb-45a9-8de6-fc37671e60f5\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:55:54.425365996Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:56:39.660041286Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-253/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-253/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: Subscript...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c9e7648dc1e0e26d2c006820b70046a7c4576357fca23729574cd01689dd6620","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508362448015360","occurred_at":"2026-03-16T11:39:08Z","failed_at":"e2e","signature_id":"882c9461e671380e208b1c1d9d84e115c1cb30212d8ec13c127fc8971fd58bb7","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0008ec3c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-cw7t2h\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000d15540\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-cw7t2h\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d15520\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-cw7t2h\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d15500\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-cw7t2h\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0008ec3c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-cw7t2h\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000d15540\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-cw7t2h\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d15520\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-cw7t2h\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d15500\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-cw7t2h\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"ddb735b2b94d3ae483bb9cd316b70930c9331b57460891c2614a752590046a72","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508362448015360","occurred_at":"2026-03-16T11:39:08Z","failed_at":"e2e","signature_id":"994235bbc238837acec0a67aab6dbcb712137ac413c6b554edcd7fff4a070217","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000de02e8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-nbrntz\" in resourcegroup=\"rg-zstream-upgrade-4-21-m85s2d\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0000df520\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-nbrntz\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-m85s2d\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0000df500\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-nbrntz\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-m85s2d\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0000df4e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-nbrntz\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-m85s2d\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000de02e8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-nbrntz\" in resourcegroup=\"rg-zstream-upgrade-4-21-m85s2d\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0000df520\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-nbrntz\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-m85s2d\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0000df500\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-nbrntz\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-m85s2d\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0000df4e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-nbrntz\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-m85s2d\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"fb54ff9cd618a641ad7ed5e7c5ac987cefebf244fe1b4d8660dd917b7ee3c5e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508362448015360","occurred_at":"2026-03-16T11:39:08Z","failed_at":"e2e","signature_id":"2f8714df25728be4d77def16f9dc5619735cbc44e3931083e4d0425478d59cf1","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d06318\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-qbf67t\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000b8fa20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-qbf67t\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b8fa00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-qbf67t\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b8f9e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-qbf67t\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d06318\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-qbf67t\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000b8fa20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-qbf67t\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b8fa00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-qbf67t\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b8f9e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-qbf67t\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"fb6acb5ed31dc7e36ae55e54797a26a19e3bbeaa8994beb93411de8b929a091a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508362448015360","occurred_at":"2026-03-16T11:39:08Z","failed_at":"e2e","signature_id":"dcfe1d6acd018b6d5ca62119f31922cf39d0d014f64f7e275c59b040f7dd8922","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.000s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0007e2620\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.000s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0007e2620\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}}],"failure_pattern_summary":{"signature_count":16,"failure_pattern_ids":["0aa9ed5d94e7167a12aa30282eff19539fc3fc473772cb96b223d420df66775f","503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","d24e138f3b84b77b0ea6793407ac1f41343d0658f0c69312eac159e4f4f7eb45"],"matched_occurrences":16,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4435/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508330667773952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4435,"pr_state":"closed","pr_sha":"4e77cea793cac38165cfba539cde69b73964b115","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:39:01Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"0ff7b7a7b2e140f5d61eb838910d936fd65d6afb4de89e15377685158a32d473","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4435/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508330667773952","occurred_at":"2026-03-16T11:39:01Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508237533253632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4444,"pr_state":"closed","pr_sha":"092800808fd635f2494b3d3e1993603df989962b","final_merged_sha":"092800808fd635f2494b3d3e1993603df989962b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T11:38:39Z"},"failed_at":["e2e"],"failed_test_count":13,"occurrences":[{"row_id":"117cd1935d9792b697e327c9921c794c9849b8e4f63a11961b2e2bbe497d1bb7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508237533253632","occurred_at":"2026-03-16T11:38:39Z","failed_at":"e2e","signature_id":"c84e186c4333e6243666c94bc7bfb9278e44cb2baf249fe2da4cb66c77036d66","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f841c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-tmwxnk\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001580e20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-tmwxnk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001580e00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-tmwxnk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001580de0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-tmwxnk\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f841c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-tmwxnk\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001580e20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-tmwxnk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001580e00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-tmwxnk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001580de0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-tmwxnk\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"20ff40412451a2ce6a13b4fb330e3c6312771dfa11ece30b770e4f2e005c6f3f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508237533253632","occurred_at":"2026-03-16T11:38:39Z","failed_at":"e2e","signature_id":"f7ac5ec26e4b2a149ba8f9559d95d48df7413fb07c2a90cf27ea13063ccb4810","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f68bd0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-s5r6vp\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a179a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-s5r6vp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a17980\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-s5r6vp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a17960\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-s5r6vp\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f68bd0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-s5r6vp\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a179a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-s5r6vp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a17980\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-s5r6vp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a17960\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-s5r6vp\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"4cfa4f57e91c6a09de0f2a020a5ed9495d99856f3537cfd40558f646d7168829","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508237533253632","occurred_at":"2026-03-16T11:38:39Z","failed_at":"e2e","signature_id":"ef7ced09d6951b0cdfa6384c20fdc0ab68be962fb17535669b336bfdf1065007","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0016e9500\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-59scjg\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31274652-8ea0-44d0-a35f-6e706cc45cef\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31274652-8ea0-44d0-a35f-6e706cc45cef\",\n \"name\": \"31274652-8ea0-44d0-a35f-6e706cc45cef\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:34:49.655222627Z\",\n \"endTime\": \"2026-03-16T12:35:29.879696907Z\",\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"Cluster provisioning failed due to multiple errors\",\n \"details\": [\n {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-105/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-105/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n },\n {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'engineering-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-59scjg/providers/Microsoft.Network/networkSecurityGroups/engineering-nsg-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-59scjg\\\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31274652-8ea0-44d0-a35f-6e706cc45cef\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31274652-8ea0-44d0-a35f-6e706cc45cef\\\",\\n \\\"name\\\": \\\"31274652-8ea0-44d0-a35f-6e706cc45cef\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:34:49.655222627Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:35:29.879696907Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-105/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-105/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n },\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'engineering-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-59scjg/providers/Microsoft.Network/networkSecurityGroups/engineering-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0016e94e0\u003e{\n msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-59scjg\\\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31274652-8ea0-44d0-a35f-6e706cc45cef\\n--------------------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0016e9500\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-59scjg\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31274652-8ea0-44d0-a35f-6e706cc45cef\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31274652-8ea0-44d0-a35f-6e706cc45cef\",\n \"name\": \"31274652-8ea0-44d0-a35f-6e706cc45cef\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:34:49.655222627Z\",\n \"endTime\": \"2026-03-16T12:35:29.879696907Z\",\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"Cluster provisioning failed due to multiple errors\",\n \"details\": [\n {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-105/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-105/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n },\n {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'engineering-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-59scjg/providers/Microsoft.Network/networkSecurityGroups/engineering-nsg-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-59scjg\\\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31274652-8ea0-44d0-a35f-6e706cc45cef\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31274652-8ea0-44d0-a35f-6e706cc45cef\\\",\\n \\\"name\\\": \\\"31274652-8ea0-44d0-a35f-6e706cc45cef\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:34:49.655222627Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:35:29.879696907Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-105/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-105/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n },\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'engineering-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-59scjg/providers/Microsoft.Network/networkSecurityGroups/engineering-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0016e94e0\u003e{\n msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-59scjg\\\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31274652-8ea0-44d0-a35f-6e706cc45cef\\n--------------------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0aa9ed5d94e7167a12aa30282eff19539fc3fc473772cb96b223d420df66775f","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"57adebe1adb6b6cf958ab27c4276962f3a99934075e699422b796fc7266426e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508237533253632","occurred_at":"2026-03-16T11:38:39Z","failed_at":"e2e","signature_id":"5ede4ed8d92153aa58e199489cdb5ec2c0cad6264abdebc346dbd998f58aeb74","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000139880\u003e: \n failed to create HCP cluster cluster-zstream-4-19-chcq8w: failed waiting for cluster=\"cluster-zstream-4-19-chcq8w\" in resourcegroup=\"rg-zstream-upgrade-4-19-rr7nj4\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5b6da78-84dd-4eba-9fbb-595df90fda1c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5b6da78-84dd-4eba-9fbb-595df90fda1c\",\n \"name\": \"c5b6da78-84dd-4eba-9fbb-595df90fda1c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:57:40.329484565Z\",\n \"endTime\": \"2026-03-16T12:58:29.778686734Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-155/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-155/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-chcq8w: failed waiting for cluster=\\\"cluster-zstream-4-19-chcq8w\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-rr7nj4\\\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5b6da78-84dd-4eba-9fbb-595df90fda1c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5b6da78-84dd-4eba-9fbb-595df90fda1c\\\",\\n \\\"name\\\": \\\"c5b6da78-84dd-4eba-9fbb-595df90fda1c\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:57:40.329484565Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:58:29.778686734Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-155/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-155/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000139860\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-19-chcq8w\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-rr7nj4\\\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5b6da78-84dd-4eba-9fbb-595df90fda1c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5b6da78-84dd-4eba-9fbb-595df90fda1c\\\",\\n \\\"name\\\": \\\"c5b6da78-84dd-4eba-9fbb-595df90fda1c\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:57:40.329484565Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:58:29.778686734Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-155/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-155/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000139880\u003e: \n failed to create HCP cluster cluster-zstream-4-19-chcq8w: failed waiting for cluster=\"cluster-zstream-4-19-chcq8w\" in resourcegroup=\"rg-zstream-upgrade-4-19-rr7nj4\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5b6da78-84dd-4eba-9fbb-595df90fda1c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5b6da78-84dd-4eba-9fbb-595df90fda1c\",\n \"name\": \"c5b6da78-84dd-4eba-9fbb-595df90fda1c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:57:40.329484565Z\",\n \"endTime\": \"2026-03-16T12:58:29.778686734Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-155/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-155/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-chcq8w: failed waiting for cluster=\\\"cluster-zstream-4-19-chcq8w\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-rr7nj4\\\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5b6da78-84dd-4eba-9fbb-595df90fda1c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5b6da78-84dd-4eba-9fbb-595df90fda1c\\\",\\n \\\"name\\\": \\\"c5b6da78-84dd-4eba-9fbb-595df90fda1c\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:57:40.329484565Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:58:29.778686734Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-155/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-155/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000139860\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-19-chcq8w\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-rr7nj4\\\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5b6da78-84dd-4eba-9fbb-595df90fda1c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5b6da78-84dd-4eba-9fbb-595df90fda1c\\\",\\n \\\"name\\\": \\\"c5b6da78-84dd-4eba-9fbb-595df90fda1c\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:57:40.329484565Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:58:29.778686734Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-155/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-155/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"60ccb91466eb5f92812b2f1e8ce8becbe1621c3fc4f21d64ce962302ea941553","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508237533253632","occurred_at":"2026-03-16T11:38:39Z","failed_at":"e2e","signature_id":"6b13cf81cf47d584ae49d5a8cb3811bafaddfd865b9f7187f41f5307df362395","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:86]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00151c900\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-mprsps\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8cc48644-1c06-4e82-89c8-139352ede7ba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8cc48644-1c06-4e82-89c8-139352ede7ba\",\n \"name\": \"8cc48644-1c06-4e82-89c8-139352ede7ba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:57:42.093796528Z\",\n \"endTime\": \"2026-03-16T12:58:29.776083252Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-mprsps\\\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8cc48644-1c06-4e82-89c8-139352ede7ba\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8cc48644-1c06-4e82-89c8-139352ede7ba\\\",\\n \\\"name\\\": \\\"8cc48644-1c06-4e82-89c8-139352ede7ba\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:57:42.093796528Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:58:29.776083252Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00151c8e0\u003e{\n msg: \"failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-mprsps\\\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8cc48644-1c06-4e82-89c8-139352ede7ba\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8cc48644-1c06-4e82-89c8-139352ede7ba\\\",\\n \\\"name\\\": \\\"8cc48644-1c06-4e82-89c8-139352ede7ba\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:57:42.093796528Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:58:29.776083252Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n---------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:86]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00151c900\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-mprsps\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8cc48644-1c06-4e82-89c8-139352ede7ba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8cc48644-1c06-4e82-89c8-139352ede7ba\",\n \"name\": \"8cc48644-1c06-4e82-89c8-139352ede7ba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:57:42.093796528Z\",\n \"endTime\": \"2026-03-16T12:58:29.776083252Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-mprsps\\\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8cc48644-1c06-4e82-89c8-139352ede7ba\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8cc48644-1c06-4e82-89c8-139352ede7ba\\\",\\n \\\"name\\\": \\\"8cc48644-1c06-4e82-89c8-139352ede7ba\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:57:42.093796528Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:58:29.776083252Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00151c8e0\u003e{\n msg: \"failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-mprsps\\\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8cc48644-1c06-4e82-89c8-139352ede7ba\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8cc48644-1c06-4e82-89c8-139352ede7ba\\\",\\n \\\"name\\\": \\\"8cc48644-1c06-4e82-89c8-139352ede7ba\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:57:42.093796528Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:58:29.776083252Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n---------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"90e9178a2e6e5cb93f9637b0e1c1ca2573bf08af51c7bb60133fa32f64fe3d6b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508237533253632","occurred_at":"2026-03-16T11:38:39Z","failed_at":"e2e","signature_id":"b427714f04a2f80ff2c3daa2675157c9c967826997a03464a66afee5d90ebab1","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010d2408\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-flqft7\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00103b140\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-flqft7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00103b120\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-flqft7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00103b100\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-flqft7\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010d2408\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-flqft7\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00103b140\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-flqft7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00103b120\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-flqft7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00103b100\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-flqft7\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"95406337e01a5c2d19b53a8f67aeacd3ab4e64bf4276b382761a8edfcef073d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508237533253632","occurred_at":"2026-03-16T11:38:39Z","failed_at":"e2e","signature_id":"d845cdd11de3ebbff479c94f70fbdeeaef8d512619445920a77c592bdd821d33","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f14240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-fwpz2w\" in resourcegroup=\"rg-zstream-upgrade-4-21-qn75gn\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000d921a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-fwpz2w\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-qn75gn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d92160\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-fwpz2w\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-qn75gn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d92140\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-fwpz2w\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-qn75gn\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f14240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-fwpz2w\" in resourcegroup=\"rg-zstream-upgrade-4-21-qn75gn\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000d921a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-fwpz2w\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-qn75gn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d92160\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-fwpz2w\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-qn75gn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d92140\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-fwpz2w\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-qn75gn\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"a10af5d9d057ebd00533b3186c35b8f6c96faeacbaee58e987fef5afa059cd9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508237533253632","occurred_at":"2026-03-16T11:38:39Z","failed_at":"e2e","signature_id":"f4212999d4e49dfe742e24809503e2546c24715aa5ff3a197fd31e4a3f5cc833","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00137c240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7kh2bf\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0010b1120\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-7kh2bf\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0010b1100\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-7kh2bf\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0010b10e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-7kh2bf\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00137c240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7kh2bf\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0010b1120\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-7kh2bf\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0010b1100\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-7kh2bf\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0010b10e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-7kh2bf\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"a60dbe9118ad13804fd73fe294a3e48704d699f131ed7dc5caf0deab7a3ef543","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508237533253632","occurred_at":"2026-03-16T11:38:39Z","failed_at":"e2e","signature_id":"4577e6af6976def115284a5be227e030e1f0379a38deee0944e15330f603c5d7","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001315380\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-b99fd4\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4da2cde2-7773-4b77-9b00-7f6387ea43fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4da2cde2-7773-4b77-9b00-7f6387ea43fd\",\n \"name\": \"4da2cde2-7773-4b77-9b00-7f6387ea43fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:57:41.183698805Z\",\n \"endTime\": \"2026-03-16T12:58:29.788885543Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-223/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-223/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-b99fd4\\\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4da2cde2-7773-4b77-9b00-7f6387ea43fd\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4da2cde2-7773-4b77-9b00-7f6387ea43fd\\\",\\n \\\"name\\\": \\\"4da2cde2-7773-4b77-9b00-7f6387ea43fd\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:57:41.183698805Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:58:29.788885543Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-223/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-223/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001315360\u003e{\n msg: \"failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-b99fd4\\\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4da2cde2-7773-4b77-9b00-7f6387ea43fd\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4da2cde2-7773-4b77-9b00-7f6387ea43fd\\\",\\n \\\"name\\\": \\\"4da2cde2-7773-4b77-9b00-7f6387ea43fd\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:57:41.183698805Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:58:29.788885543Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-223/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-223/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: Subscript...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001315380\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-b99fd4\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4da2cde2-7773-4b77-9b00-7f6387ea43fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4da2cde2-7773-4b77-9b00-7f6387ea43fd\",\n \"name\": \"4da2cde2-7773-4b77-9b00-7f6387ea43fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:57:41.183698805Z\",\n \"endTime\": \"2026-03-16T12:58:29.788885543Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-223/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-223/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-b99fd4\\\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4da2cde2-7773-4b77-9b00-7f6387ea43fd\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4da2cde2-7773-4b77-9b00-7f6387ea43fd\\\",\\n \\\"name\\\": \\\"4da2cde2-7773-4b77-9b00-7f6387ea43fd\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:57:41.183698805Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:58:29.788885543Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-223/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-223/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001315360\u003e{\n msg: \"failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-b99fd4\\\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4da2cde2-7773-4b77-9b00-7f6387ea43fd\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4da2cde2-7773-4b77-9b00-7f6387ea43fd\\\",\\n \\\"name\\\": \\\"4da2cde2-7773-4b77-9b00-7f6387ea43fd\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:57:41.183698805Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:58:29.788885543Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-223/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-223/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: Subscript...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a636725b28f076fd0f421d001ca123b2efbee701c40b8dcc8e735d3e3f015a7a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508237533253632","occurred_at":"2026-03-16T11:38:39Z","failed_at":"e2e","signature_id":"0517645541cc23b0d17037b02219d7dc02ef0ac10dfcf818de4b4d17f008d54a","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00136c1b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-lctbf2\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001418ec0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-lctbf2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001418ea0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-lctbf2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001418e80\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-lctbf2\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00136c1b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-lctbf2\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001418ec0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-lctbf2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001418ea0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-lctbf2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001418e80\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-lctbf2\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"d1308a5666d2d37b9de48d628670f8166dcefc3c6aa9f9af185a22578ed04093","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508237533253632","occurred_at":"2026-03-16T11:38:39Z","failed_at":"e2e","signature_id":"5c5ab374f2d5204cf43f1e9cfe811f9689ccd23b69af3a091b06996ca698d322","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0013b2228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-4dk9v7\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007bdbe0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-4dk9v7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007bdbc0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-4dk9v7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007bdba0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-4dk9v7\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0013b2228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-4dk9v7\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007bdbe0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-4dk9v7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007bdbc0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-4dk9v7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007bdba0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-4dk9v7\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"f363e8bb853a723556eca74985fb7b5e0fe061209bbf538062f34da2176b96be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508237533253632","occurred_at":"2026-03-16T11:38:39Z","failed_at":"e2e","signature_id":"b1812d793107ce5ebe154c88d3b63a1401f5040a2673333d4810f9dcff5fbee9","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ca8438\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-689h6l\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0011a1140\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-689h6l\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0011a1120\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-689h6l\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0011a1100\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-689h6l\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ca8438\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-689h6l\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0011a1140\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-689h6l\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0011a1120\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-689h6l\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0011a1100\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-689h6l\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"f7adcda0e6f0b6938c7780694b0d5b5871e3b234e796357eb21c9d7cff49d073","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508237533253632","occurred_at":"2026-03-16T11:38:39Z","failed_at":"e2e","signature_id":"734989ede785d477d2bdf7715db572856e256b7c7033ebf3cae65337e514cbca","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00169b6a0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-ctj74c\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5339c1b7-6a5a-4939-b1c7-d8c913fdb0e7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5339c1b7-6a5a-4939-b1c7-d8c913fdb0e7\",\n \"name\": \"5339c1b7-6a5a-4939-b1c7-d8c913fdb0e7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:35:38.355734247Z\",\n \"endTime\": \"2026-03-16T12:36:31.750918101Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-ctj74c\\\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5339c1b7-6a5a-4939-b1c7-d8c913fdb0e7\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5339c1b7-6a5a-4939-b1c7-d8c913fdb0e7\\\",\\n \\\"name\\\": \\\"5339c1b7-6a5a-4939-b1c7-d8c913fdb0e7\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:35:38.355734247Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:36:31.750918101Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00169b680\u003e{\n msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-ctj74c\\\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5339c1b7-6a5a-4939-b1c7-d8c913fdb0e7\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5339c1b7-6a5a-4939-b1c7-d8c913fdb0e7\\\",\\n \\\"name\\\": \\\"5339c1b7-6a5a-4939-b1c7-d8c913fdb0e7\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:35:38.355734247Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:36:31.750918101Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n----------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00169b6a0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-ctj74c\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5339c1b7-6a5a-4939-b1c7-d8c913fdb0e7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5339c1b7-6a5a-4939-b1c7-d8c913fdb0e7\",\n \"name\": \"5339c1b7-6a5a-4939-b1c7-d8c913fdb0e7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:35:38.355734247Z\",\n \"endTime\": \"2026-03-16T12:36:31.750918101Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-ctj74c\\\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5339c1b7-6a5a-4939-b1c7-d8c913fdb0e7\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5339c1b7-6a5a-4939-b1c7-d8c913fdb0e7\\\",\\n \\\"name\\\": \\\"5339c1b7-6a5a-4939-b1c7-d8c913fdb0e7\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:35:38.355734247Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:36:31.750918101Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00169b680\u003e{\n msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-ctj74c\\\" to finish creating: GET https://rp.j3253632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5339c1b7-6a5a-4939-b1c7-d8c913fdb0e7\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5339c1b7-6a5a-4939-b1c7-d8c913fdb0e7\\\",\\n \\\"name\\\": \\\"5339c1b7-6a5a-4939-b1c7-d8c913fdb0e7\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:35:38.355734247Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:36:31.750918101Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n----------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":13,"failure_pattern_ids":["0aa9ed5d94e7167a12aa30282eff19539fc3fc473772cb96b223d420df66775f","503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761"],"matched_occurrences":13,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4446/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508205442633728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4446,"pr_state":"closed","pr_sha":"5ed9c72a6262f923de3a314d902f2b394fd3765b","final_merged_sha":"1d16d503d07580c2671790287caed928be96c542","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:38:31Z"},"failed_at":["e2e"],"failed_test_count":13,"occurrences":[{"row_id":"0f55e20a875ede993933143963b8b994e7af353de1418b7f5cf017e1b9265a0e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4446/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508205442633728","occurred_at":"2026-03-16T11:38:31Z","failed_at":"e2e","signature_id":"1d8385ec0ee2e8d39d819b9e2cc31544bed2f70c758e8a2a7c2d1eab6b7ae923","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000a488b8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-jj6k4s\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000507ca0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-jj6k4s\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000507c80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-jj6k4s\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000507c60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-jj6k4s\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000a488b8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-jj6k4s\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000507ca0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-jj6k4s\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000507c80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-jj6k4s\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000507c60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-jj6k4s\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"16848065e65b72a0238932127fe95cf221114bc52d1188dafb7ed514994c1326","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4446/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508205442633728","occurred_at":"2026-03-16T11:38:31Z","failed_at":"e2e","signature_id":"5c43756a94e4b3f9e43420c43f3a1defaad271c456f944a236af9198504b25bd","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00176a210\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-c8v8vx\" in resourcegroup=\"rg-zstream-upgrade-4-21-6h9w42\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00014bd20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-c8v8vx\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-6h9w42\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00014bd00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-c8v8vx\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-6h9w42\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00014bce0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-c8v8vx\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-6h9w42\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00176a210\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-c8v8vx\" in resourcegroup=\"rg-zstream-upgrade-4-21-6h9w42\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00014bd20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-c8v8vx\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-6h9w42\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00014bd00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-c8v8vx\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-6h9w42\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00014bce0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-c8v8vx\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-6h9w42\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"1c339fbf43e18bab3ed9d41c877b55c33bb0a8b3cedbe5ec55675ef42f437a62","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4446/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508205442633728","occurred_at":"2026-03-16T11:38:31Z","failed_at":"e2e","signature_id":"7fe0b93b61cc80f67ebf5c60265cb341961c77fdde4a584d72d1d4466d8a737c","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001130120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-wtq2pg\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a04fc0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-wtq2pg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a04f80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-wtq2pg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a04f60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-wtq2pg\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001130120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-wtq2pg\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a04fc0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-wtq2pg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a04f80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-wtq2pg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a04f60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-wtq2pg\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"408a878adacf4455f4db5207c026e53f209d7cbde9d5771999f26d11308379cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4446/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508205442633728","occurred_at":"2026-03-16T11:38:31Z","failed_at":"e2e","signature_id":"e0e0916d3fb172334814db643d43b90c6458ca722af61bacadeffcf1447dc2a8","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.005s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00075a4a0\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.005s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00075a4a0\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}},{"row_id":"56620e1efdaf789981f7ac6cf6390d9f316cc641c0523fd28892067a64f5b140","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4446/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508205442633728","occurred_at":"2026-03-16T11:38:31Z","failed_at":"e2e","signature_id":"450c6f24138c3b372cb4e9a980f3bffc092f45a4a855dceb414ff2fb664f3be6","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00173e1f8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-2df2bg\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000ed2ea0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-2df2bg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ed2e80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-2df2bg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ed2e60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-2df2bg\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00173e1f8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-2df2bg\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000ed2ea0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-2df2bg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ed2e80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-2df2bg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ed2e60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-2df2bg\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"69af2283f06a0b08be11512713517c40283f4aa1506b2e31ef9b3fb818d7f530","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4446/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508205442633728","occurred_at":"2026-03-16T11:38:31Z","failed_at":"e2e","signature_id":"003427203deeb67eb6cd2ee5008c087404694fc662fb8a8b68b9dee8671177f5","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:86]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00080e580\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-xfg9dt\" to finish creating: GET https://rp.j2633728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aea717e8-25cf-4ca9-bda1-ba9361a650ad\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aea717e8-25cf-4ca9-bda1-ba9361a650ad\",\n \"name\": \"aea717e8-25cf-4ca9-bda1-ba9361a650ad\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T15:02:20.561460313Z\",\n \"endTime\": \"2026-03-16T15:03:24.977979414Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-144/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-144/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-xfg9dt\\\" to finish creating: GET https://rp.j2633728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aea717e8-25cf-4ca9-bda1-ba9361a650ad\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aea717e8-25cf-4ca9-bda1-ba9361a650ad\\\",\\n \\\"name\\\": \\\"aea717e8-25cf-4ca9-bda1-ba9361a650ad\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T15:02:20.561460313Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T15:03:24.977979414Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-144/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-144/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00080e540\u003e{\n msg: \"failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-xfg9dt\\\" to finish creating: GET https://rp.j2633728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aea717e8-25cf-4ca9-bda1-ba9361a650ad\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aea717e8-25cf-4ca9-bda1-ba9361a650ad\\\",\\n \\\"name\\\": \\\"aea717e8-25cf-4ca9-bda1-ba9361a650ad\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T15:02:20.561460313Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T15:03:24.977979414Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-144/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-144/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:86]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00080e580\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-xfg9dt\" to finish creating: GET https://rp.j2633728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aea717e8-25cf-4ca9-bda1-ba9361a650ad\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aea717e8-25cf-4ca9-bda1-ba9361a650ad\",\n \"name\": \"aea717e8-25cf-4ca9-bda1-ba9361a650ad\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T15:02:20.561460313Z\",\n \"endTime\": \"2026-03-16T15:03:24.977979414Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-144/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-144/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-xfg9dt\\\" to finish creating: GET https://rp.j2633728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aea717e8-25cf-4ca9-bda1-ba9361a650ad\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aea717e8-25cf-4ca9-bda1-ba9361a650ad\\\",\\n \\\"name\\\": \\\"aea717e8-25cf-4ca9-bda1-ba9361a650ad\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T15:02:20.561460313Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T15:03:24.977979414Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-144/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-144/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00080e540\u003e{\n msg: \"failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-xfg9dt\\\" to finish creating: GET https://rp.j2633728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aea717e8-25cf-4ca9-bda1-ba9361a650ad\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aea717e8-25cf-4ca9-bda1-ba9361a650ad\\\",\\n \\\"name\\\": \\\"aea717e8-25cf-4ca9-bda1-ba9361a650ad\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T15:02:20.561460313Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T15:03:24.977979414Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-144/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-144/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"91a87faa759b83c72cdedef3891e183011e02d8c3f7570862d1416d21a828776","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4446/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508205442633728","occurred_at":"2026-03-16T11:38:31Z","failed_at":"e2e","signature_id":"ca49c10191fcdfc7af9aebf2dbe4a68e56385a6c795d08953d2ff3303e5954e4","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00097c6f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-5ktksf\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000ab75c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-5ktksf\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ab75a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-5ktksf\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ab7580\u003e{\n msg: \"failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-5ktksf\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00097c6f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-5ktksf\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000ab75c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-5ktksf\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ab75a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-5ktksf\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ab7580\u003e{\n msg: \"failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-5ktksf\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"a9b5a542a8fe1d9f53cce67d9635d1cf684d6d2e73a3c656c4284f873c3809d6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4446/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508205442633728","occurred_at":"2026-03-16T11:38:31Z","failed_at":"e2e","signature_id":"8a5d7c0d70d22257bd0c28b6d1ebfc32b443c2d13f1ab31d51b45e079cf61b82","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0016620c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-n8ww9v\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0012873c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-n8ww9v\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0012873a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-n8ww9v\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001287380\u003e{\n msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-n8ww9v\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0016620c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-n8ww9v\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0012873c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-n8ww9v\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0012873a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-n8ww9v\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001287380\u003e{\n msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-n8ww9v\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"ab487c44936288d0b7f4be1affd25e993d5a561dfdf63676f40c96dc710d1caa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4446/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508205442633728","occurred_at":"2026-03-16T11:38:31Z","failed_at":"e2e","signature_id":"19b3fa65bc61996e8a6a7f0c44f522b7246ab063bd5e179265ccf9db8bfd4417","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000fdc3f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-kmmt9r\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00088a260\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-kmmt9r\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00088a240\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-kmmt9r\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00088a200\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-kmmt9r\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000fdc3f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-kmmt9r\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00088a260\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-kmmt9r\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00088a240\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-kmmt9r\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00088a200\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-kmmt9r\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"c2948a5c18cc1579c5d5d0139f0a62c57300bff8c6e30ba5e3e031511f374c1b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4446/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508205442633728","occurred_at":"2026-03-16T11:38:31Z","failed_at":"e2e","signature_id":"85fa41084bd28ad633f5c8c207eef23c675a84427ac78e4595e36fb0c05e4fff","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d38270\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-jbs926\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000fe4820\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-jbs926\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000fe47e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-jbs926\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0013f9520\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-jbs926\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d38270\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-jbs926\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000fe4820\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-jbs926\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000fe47e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-jbs926\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0013f9520\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-jbs926\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"cf676232b3f072479092ae99702da4f87c092165b057ff94502d2754e3b29ccb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4446/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508205442633728","occurred_at":"2026-03-16T11:38:31Z","failed_at":"e2e","signature_id":"48ba9c84ddc3697e21d055fcbb112def740700e8ce4fb606d291df26300bff5a","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e64258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-q842nj\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000b2fb60\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-q842nj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b2fb20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-q842nj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b2fae0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-q842nj\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e64258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-q842nj\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000b2fb60\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-q842nj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b2fb20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-q842nj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b2fae0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-q842nj\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"d45ad5daff19e4501dd601ea3578862cef114b90d0b2af482b9ff6e892e1fb0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4446/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508205442633728","occurred_at":"2026-03-16T11:38:31Z","failed_at":"e2e","signature_id":"c1f3ebb0ce46ed2722a5f26c06ec52076ff5d090935723fa6f92a1eb8ee9edf7","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b35bc0\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-2ttk57\" to finish creating: GET https://rp.j2633728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6031911-24e5-4ffa-a97f-ee993d430b06\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6031911-24e5-4ffa-a97f-ee993d430b06\",\n \"name\": \"a6031911-24e5-4ffa-a97f-ee993d430b06\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:28:14.441998363Z\",\n \"endTime\": \"2026-03-16T14:29:08.761109691Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-121/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-121/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: 429\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"429\\\",\\n \\\"message\\\": \\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 5 seconds.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-2ttk57\\\" to finish creating: GET https://rp.j2633728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6031911-24e5-4ffa-a97f-ee993d430b06\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6031911-24e5-4ffa-a97f-ee993d430b06\\\",\\n \\\"name\\\": \\\"a6031911-24e5-4ffa-a97f-ee993d430b06\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:28:14.441998363Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:29:08.761109691Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-121/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-121/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: 429\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"429\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 5 seconds.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012455f0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"be961142-a154-4bed-8f44-db4c4e95ae55\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 14:29:14 GMT\",\n ],\n \"Content-Length\": [\"1382\"],\n \"X-Envoy-Upstream-Service-Time\": [\"53\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000b35600\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 97, 54, 48, 51, 49, 57, 49, 49, 45, 50, 52, 101, 53, 45, 52, 102, 102, 97, 45, 97, 57, 55, 102, 45, 101, 101, 57, 57, 51, 100, 52, 51, 48, 98, 48, 54, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 97, 54, 48, 51, 49, 57, 49, 49, 45, 50, 52, 101, 53, 45, 52, 102, 102, 97, 45, 97, 57, 55, 102, 45, 101, 101, 57, 57, 51, 100, 52, 51, 48, 98, 48, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b35bc0\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-2ttk57\" to finish creating: GET https://rp.j2633728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6031911-24e5-4ffa-a97f-ee993d430b06\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6031911-24e5-4ffa-a97f-ee993d430b06\",\n \"name\": \"a6031911-24e5-4ffa-a97f-ee993d430b06\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:28:14.441998363Z\",\n \"endTime\": \"2026-03-16T14:29:08.761109691Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-121/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-121/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: 429\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"429\\\",\\n \\\"message\\\": \\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 5 seconds.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-2ttk57\\\" to finish creating: GET https://rp.j2633728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6031911-24e5-4ffa-a97f-ee993d430b06\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6031911-24e5-4ffa-a97f-ee993d430b06\\\",\\n \\\"name\\\": \\\"a6031911-24e5-4ffa-a97f-ee993d430b06\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:28:14.441998363Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:29:08.761109691Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-121/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-121/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: 429\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"429\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 5 seconds.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012455f0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"be961142-a154-4bed-8f44-db4c4e95ae55\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 14:29:14 GMT\",\n ],\n \"Content-Length\": [\"1382\"],\n \"X-Envoy-Upstream-Service-Time\": [\"53\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000b35600\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 97, 54, 48, 51, 49, 57, 49, 49, 45, 50, 52, 101, 53, 45, 52, 102, 102, 97, 45, 97, 57, 55, 102, 45, 101, 101, 57, 57, 51, 100, 52, 51, 48, 98, 48, 54, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 97, 54, 48, 51, 49, 57, 49, 49, 45, 50, 52, 101, 53, 45, 52, 102, 102, 97, 45, 97, 57, 55, 102, 45, 101, 101, 57, 57, 51, 100, 52, 51, 48, 98, 48, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24e6e981cfe355409e7b9a2e92ed5ca47951bdce335ac59e3a3c469749129106","failure_pattern":"ERROR CODE: InternalServerError; detail code 429; detail message Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 5 seconds.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"eac7939ce1ab60c12dfaa932d1bc41251ac8cdb228545a0914a7c19a2c4fa331","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4446/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508205442633728","occurred_at":"2026-03-16T11:38:31Z","failed_at":"e2e","signature_id":"4188d4a06db73f280041561100f5fbb53dd46d36301e0431e2a20b6932e2ef2a","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0016e22a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-fk5jp8\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00071b960\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fk5jp8\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00071b940\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fk5jp8\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00071b920\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fk5jp8\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0016e22a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-fk5jp8\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00071b960\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fk5jp8\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00071b940\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fk5jp8\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00071b920\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fk5jp8\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":13,"failure_pattern_ids":["24e6e981cfe355409e7b9a2e92ed5ca47951bdce335ac59e3a3c469749129106","503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7"],"matched_occurrences":13,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4450/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508141848596480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4450,"pr_state":"closed","pr_sha":"cb1d0d4acb67a0b6c1fd3be06011893b5e4b7b83","final_merged_sha":"cb1d0d4acb67a0b6c1fd3be06011893b5e4b7b83","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T11:38:16Z"},"failed_at":["e2e"],"failed_test_count":8,"occurrences":[{"row_id":"082c13c21981d1fb0490367346b97fa666adf7b1802e1f9169f692f74bc4934c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4450/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508141848596480","occurred_at":"2026-03-16T11:38:16Z","failed_at":"e2e","signature_id":"28f227338d91c59bfcd3d82ff70fba49a9751f13c6990c1175f59c664b46ceaa","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000597540\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-pdhhqw\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3125e7dd-2e11-4b1a-a107-ac55289b8b19\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3125e7dd-2e11-4b1a-a107-ac55289b8b19\",\n \"name\": \"3125e7dd-2e11-4b1a-a107-ac55289b8b19\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T13:09:12.515669684Z\",\n \"endTime\": \"2026-03-16T13:10:09.05502246Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-297/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-297/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-pdhhqw\\\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3125e7dd-2e11-4b1a-a107-ac55289b8b19\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3125e7dd-2e11-4b1a-a107-ac55289b8b19\\\",\\n \\\"name\\\": \\\"3125e7dd-2e11-4b1a-a107-ac55289b8b19\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T13:09:12.515669684Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T13:10:09.05502246Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-297/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-297/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000597520\u003e{\n msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-pdhhqw\\\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3125e7dd-2e11-4b1a-a107-ac55289b8b19\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3125e7dd-2e11-4b1a-a107-ac55289b8b19\\\",\\n \\\"name\\\": \\\"3125e7dd-2e11-4b1a-a107-ac55289b8b19\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T13:09:12.515669684Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T13:10:09.05502246Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-297/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-297/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-----------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000597540\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-pdhhqw\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3125e7dd-2e11-4b1a-a107-ac55289b8b19\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3125e7dd-2e11-4b1a-a107-ac55289b8b19\",\n \"name\": \"3125e7dd-2e11-4b1a-a107-ac55289b8b19\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T13:09:12.515669684Z\",\n \"endTime\": \"2026-03-16T13:10:09.05502246Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-297/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-297/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-pdhhqw\\\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3125e7dd-2e11-4b1a-a107-ac55289b8b19\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3125e7dd-2e11-4b1a-a107-ac55289b8b19\\\",\\n \\\"name\\\": \\\"3125e7dd-2e11-4b1a-a107-ac55289b8b19\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T13:09:12.515669684Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T13:10:09.05502246Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-297/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-297/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000597520\u003e{\n msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-pdhhqw\\\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3125e7dd-2e11-4b1a-a107-ac55289b8b19\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3125e7dd-2e11-4b1a-a107-ac55289b8b19\\\",\\n \\\"name\\\": \\\"3125e7dd-2e11-4b1a-a107-ac55289b8b19\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T13:09:12.515669684Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T13:10:09.05502246Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-297/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-297/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-----------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0e0baef5efb592e20dc9990a157d44ebc44bf21c3903239bd6a70164a04b9542","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4450/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508141848596480","occurred_at":"2026-03-16T11:38:16Z","failed_at":"e2e","signature_id":"13b2a8e206e4efb14f85b6dd626e5ec1799fd085cd56ddb2f02f47ec6edefa92","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000921960\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-mcrpkg\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f6cd8c6f-2f9a-46ff-8baa-01ea33049823\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f6cd8c6f-2f9a-46ff-8baa-01ea33049823\",\n \"name\": \"f6cd8c6f-2f9a-46ff-8baa-01ea33049823\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T13:09:25.554283148Z\",\n \"endTime\": \"2026-03-16T13:10:09.075846622Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-100/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-100/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-mcrpkg\\\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f6cd8c6f-2f9a-46ff-8baa-01ea33049823\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f6cd8c6f-2f9a-46ff-8baa-01ea33049823\\\",\\n \\\"name\\\": \\\"f6cd8c6f-2f9a-46ff-8baa-01ea33049823\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T13:09:25.554283148Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T13:10:09.075846622Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-100/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-100/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000921920\u003e{\n msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-mcrpkg\\\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f6cd8c6f-2f9a-46ff-8baa-01ea33049823\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f6cd8c6f-2f9a-46ff-8baa-01ea33049823\\\",\\n \\\"name\\\": \\\"f6cd8c6f-2f9a-46ff-8baa-01ea33049823\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T13:09:25.554283148Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T13:10:09.075846622Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-100/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-100/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000921960\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-mcrpkg\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f6cd8c6f-2f9a-46ff-8baa-01ea33049823\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f6cd8c6f-2f9a-46ff-8baa-01ea33049823\",\n \"name\": \"f6cd8c6f-2f9a-46ff-8baa-01ea33049823\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T13:09:25.554283148Z\",\n \"endTime\": \"2026-03-16T13:10:09.075846622Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-100/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-100/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-mcrpkg\\\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f6cd8c6f-2f9a-46ff-8baa-01ea33049823\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f6cd8c6f-2f9a-46ff-8baa-01ea33049823\\\",\\n \\\"name\\\": \\\"f6cd8c6f-2f9a-46ff-8baa-01ea33049823\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T13:09:25.554283148Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T13:10:09.075846622Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-100/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-100/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000921920\u003e{\n msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-mcrpkg\\\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f6cd8c6f-2f9a-46ff-8baa-01ea33049823\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f6cd8c6f-2f9a-46ff-8baa-01ea33049823\\\",\\n \\\"name\\\": \\\"f6cd8c6f-2f9a-46ff-8baa-01ea33049823\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T13:09:25.554283148Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T13:10:09.075846622Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-100/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-100/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"181e0cf58f3a41288e4abd72b42b1fb46518f09df4b4e425e6c73cd3abeb4204","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4450/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508141848596480","occurred_at":"2026-03-16T11:38:16Z","failed_at":"e2e","signature_id":"fcd4a01102e02b7cb4d1f820dcc4104d975e48f560b3973c63a6f97895fc293b","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0010ca540\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-6bxc98\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02ff7b81-7cfd-4397-963c-dfb697e5c84b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02ff7b81-7cfd-4397-963c-dfb697e5c84b\",\n \"name\": \"02ff7b81-7cfd-4397-963c-dfb697e5c84b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T13:09:22.548426273Z\",\n \"endTime\": \"2026-03-16T13:10:09.038847011Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-5/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-5/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-6bxc98\\\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02ff7b81-7cfd-4397-963c-dfb697e5c84b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02ff7b81-7cfd-4397-963c-dfb697e5c84b\\\",\\n \\\"name\\\": \\\"02ff7b81-7cfd-4397-963c-dfb697e5c84b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T13:09:22.548426273Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T13:10:09.038847011Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-5/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-5/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0010ca520\u003e{\n msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-6bxc98\\\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02ff7b81-7cfd-4397-963c-dfb697e5c84b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02ff7b81-7cfd-4397-963c-dfb697e5c84b\\\",\\n \\\"name\\\": \\\"02ff7b81-7cfd-4397-963c-dfb697e5c84b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T13:09:22.548426273Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T13:10:09.038847011Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-5/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-5/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-----------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0010ca540\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-6bxc98\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02ff7b81-7cfd-4397-963c-dfb697e5c84b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02ff7b81-7cfd-4397-963c-dfb697e5c84b\",\n \"name\": \"02ff7b81-7cfd-4397-963c-dfb697e5c84b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T13:09:22.548426273Z\",\n \"endTime\": \"2026-03-16T13:10:09.038847011Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-5/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-5/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-6bxc98\\\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02ff7b81-7cfd-4397-963c-dfb697e5c84b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02ff7b81-7cfd-4397-963c-dfb697e5c84b\\\",\\n \\\"name\\\": \\\"02ff7b81-7cfd-4397-963c-dfb697e5c84b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T13:09:22.548426273Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T13:10:09.038847011Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-5/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-5/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0010ca520\u003e{\n msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-6bxc98\\\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02ff7b81-7cfd-4397-963c-dfb697e5c84b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02ff7b81-7cfd-4397-963c-dfb697e5c84b\\\",\\n \\\"name\\\": \\\"02ff7b81-7cfd-4397-963c-dfb697e5c84b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T13:09:22.548426273Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T13:10:09.038847011Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-5/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-5/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-----------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3e5107c55ecd416bca192037a54a0173ee63c00b052b4e318d2ce2444288cb21","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4450/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508141848596480","occurred_at":"2026-03-16T11:38:16Z","failed_at":"e2e","signature_id":"b7a5871059fbea7126f4d31d8c0eb1cfcaa407ab7abef35145f5e407f7e5de5b","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b28c20\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-gjc7zg\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/635b1c05-4f04-4ea3-8f1c-3caef8df7984\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/635b1c05-4f04-4ea3-8f1c-3caef8df7984\",\n \"name\": \"635b1c05-4f04-4ea3-8f1c-3caef8df7984\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T13:09:13.599864638Z\",\n \"endTime\": \"2026-03-16T13:10:08.97808366Z\",\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"Cluster provisioning failed due to multiple errors\",\n \"details\": [\n {\n \"code\": \"InternalServerError\",\n \"message\": \"Failed to get virtual network 'customer-vnet-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-gjc7zg/providers/Microsoft.Network/virtualNetworks/customer-vnet-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n },\n {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-64/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-64/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-gjc7zg\\\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/635b1c05-4f04-4ea3-8f1c-3caef8df7984\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/635b1c05-4f04-4ea3-8f1c-3caef8df7984\\\",\\n \\\"name\\\": \\\"635b1c05-4f04-4ea3-8f1c-3caef8df7984\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T13:09:13.599864638Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T13:10:08.97808366Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-gjc7zg/providers/Microsoft.Network/virtualNetworks/customer-vnet-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n },\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-64/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-64/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000b28c00\u003e{\n msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-gjc7zg\\\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/635b1c05-4f04-4ea3-8f1c-3ca...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b28c20\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-gjc7zg\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/635b1c05-4f04-4ea3-8f1c-3caef8df7984\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/635b1c05-4f04-4ea3-8f1c-3caef8df7984\",\n \"name\": \"635b1c05-4f04-4ea3-8f1c-3caef8df7984\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T13:09:13.599864638Z\",\n \"endTime\": \"2026-03-16T13:10:08.97808366Z\",\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"Cluster provisioning failed due to multiple errors\",\n \"details\": [\n {\n \"code\": \"InternalServerError\",\n \"message\": \"Failed to get virtual network 'customer-vnet-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-gjc7zg/providers/Microsoft.Network/virtualNetworks/customer-vnet-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n },\n {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-64/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-64/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-gjc7zg\\\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/635b1c05-4f04-4ea3-8f1c-3caef8df7984\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/635b1c05-4f04-4ea3-8f1c-3caef8df7984\\\",\\n \\\"name\\\": \\\"635b1c05-4f04-4ea3-8f1c-3caef8df7984\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T13:09:13.599864638Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T13:10:08.97808366Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-gjc7zg/providers/Microsoft.Network/virtualNetworks/customer-vnet-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n },\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-64/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-64/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000b28c00\u003e{\n msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-gjc7zg\\\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/635b1c05-4f04-4ea3-8f1c-3ca...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0aa9ed5d94e7167a12aa30282eff19539fc3fc473772cb96b223d420df66775f","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"80ef3089b0fa6778d07650bcf9a69a25fd0915b3d9dcd68bddd38140f2306e61","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4450/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508141848596480","occurred_at":"2026-03-16T11:38:16Z","failed_at":"e2e","signature_id":"c841d8910dd4bc96a3676dbc6543c6c938f9209b7eb332fad3de09445b04adb9","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000ea58e0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-4f47jw\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/657687f8-0b2a-4bf5-98a8-7b0b31b9e3dd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/657687f8-0b2a-4bf5-98a8-7b0b31b9e3dd\",\n \"name\": \"657687f8-0b2a-4bf5-98a8-7b0b31b9e3dd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T13:09:22.698540737Z\",\n \"endTime\": \"2026-03-16T13:10:09.035865816Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-62/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-62/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster ea-cluster: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-4f47jw\\\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/657687f8-0b2a-4bf5-98a8-7b0b31b9e3dd\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/657687f8-0b2a-4bf5-98a8-7b0b31b9e3dd\\\",\\n \\\"name\\\": \\\"657687f8-0b2a-4bf5-98a8-7b0b31b9e3dd\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T13:09:22.698540737Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T13:10:09.035865816Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-62/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-62/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000ea58c0\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-4f47jw\\\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/657687f8-0b2a-4bf5-98a8-7b0b31b9e3dd\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/657687f8-0b2a-4bf5-98a8-7b0b31b9e3dd\\\",\\n \\\"name\\\": \\\"657687f8-0b2a-4bf5-98a8-7b0b31b9e3dd\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T13:09:22.698540737Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T13:10:09.035865816Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-62/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-62/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000ea58e0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-4f47jw\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/657687f8-0b2a-4bf5-98a8-7b0b31b9e3dd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/657687f8-0b2a-4bf5-98a8-7b0b31b9e3dd\",\n \"name\": \"657687f8-0b2a-4bf5-98a8-7b0b31b9e3dd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T13:09:22.698540737Z\",\n \"endTime\": \"2026-03-16T13:10:09.035865816Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-62/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-62/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster ea-cluster: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-4f47jw\\\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/657687f8-0b2a-4bf5-98a8-7b0b31b9e3dd\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/657687f8-0b2a-4bf5-98a8-7b0b31b9e3dd\\\",\\n \\\"name\\\": \\\"657687f8-0b2a-4bf5-98a8-7b0b31b9e3dd\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T13:09:22.698540737Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T13:10:09.035865816Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-62/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-62/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000ea58c0\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-4f47jw\\\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/657687f8-0b2a-4bf5-98a8-7b0b31b9e3dd\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/657687f8-0b2a-4bf5-98a8-7b0b31b9e3dd\\\",\\n \\\"name\\\": \\\"657687f8-0b2a-4bf5-98a8-7b0b31b9e3dd\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T13:09:22.698540737Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T13:10:09.035865816Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-62/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-62/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"86048c9153d527aa5ead60fc7510a55a8766b0873d78e6c2390f6e63c0e4410e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4450/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508141848596480","occurred_at":"2026-03-16T11:38:16Z","failed_at":"e2e","signature_id":"bd0ca9d5fd9e03abadeef12aeb470c3da97382ce084e7094794e414070e5d0c2","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a26580\u003e: \n failed to create HCP cluster ea-listv7w2gg: failed waiting for cluster=\"ea-listv7w2gg\" in resourcegroup=\"ea-list-rx6tlz\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cea25ca-7954-433a-ae78-8d863b6ed26c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cea25ca-7954-433a-ae78-8d863b6ed26c\",\n \"name\": \"9cea25ca-7954-433a-ae78-8d863b6ed26c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T13:09:21.953309402Z\",\n \"endTime\": \"2026-03-16T13:10:09.037665494Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster ea-listv7w2gg: failed waiting for cluster=\\\"ea-listv7w2gg\\\" in resourcegroup=\\\"ea-list-rx6tlz\\\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cea25ca-7954-433a-ae78-8d863b6ed26c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cea25ca-7954-433a-ae78-8d863b6ed26c\\\",\\n \\\"name\\\": \\\"9cea25ca-7954-433a-ae78-8d863b6ed26c\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T13:09:21.953309402Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T13:10:09.037665494Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a26560\u003e{\n msg: \"failed waiting for cluster=\\\"ea-listv7w2gg\\\" in resourcegroup=\\\"ea-list-rx6tlz\\\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cea25ca-7954-433a-ae78-8d863b6ed26c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cea25ca-7954-433a-ae78-8d863b6ed26c\\\",\\n \\\"name\\\": \\\"9cea25ca-7954-433a-ae78-8d863b6ed26c\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T13:09:21.953309402Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T13:10:09.037665494Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"cod...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a26580\u003e: \n failed to create HCP cluster ea-listv7w2gg: failed waiting for cluster=\"ea-listv7w2gg\" in resourcegroup=\"ea-list-rx6tlz\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cea25ca-7954-433a-ae78-8d863b6ed26c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cea25ca-7954-433a-ae78-8d863b6ed26c\",\n \"name\": \"9cea25ca-7954-433a-ae78-8d863b6ed26c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T13:09:21.953309402Z\",\n \"endTime\": \"2026-03-16T13:10:09.037665494Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster ea-listv7w2gg: failed waiting for cluster=\\\"ea-listv7w2gg\\\" in resourcegroup=\\\"ea-list-rx6tlz\\\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cea25ca-7954-433a-ae78-8d863b6ed26c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cea25ca-7954-433a-ae78-8d863b6ed26c\\\",\\n \\\"name\\\": \\\"9cea25ca-7954-433a-ae78-8d863b6ed26c\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T13:09:21.953309402Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T13:10:09.037665494Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a26560\u003e{\n msg: \"failed waiting for cluster=\\\"ea-listv7w2gg\\\" in resourcegroup=\\\"ea-list-rx6tlz\\\" to finish creating: GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cea25ca-7954-433a-ae78-8d863b6ed26c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cea25ca-7954-433a-ae78-8d863b6ed26c\\\",\\n \\\"name\\\": \\\"9cea25ca-7954-433a-ae78-8d863b6ed26c\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T13:09:21.953309402Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T13:10:09.037665494Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"cod...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c086cd770cb44c6fa425e4273c1eea356a4a7748fde7a6784b416d8cc2067186","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4450/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508141848596480","occurred_at":"2026-03-16T11:38:16Z","failed_at":"e2e","signature_id":"31fa2ab3fd0ea005b0d35378511f6068c279de541159a495f5a60fd01c8130b0","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.007s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0008e9650\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.007s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0008e9650\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}},{"row_id":"efd356b73774af26e23b371929405a178e02c61941da925b429283543062cca1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4450/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508141848596480","occurred_at":"2026-03-16T11:38:16Z","failed_at":"e2e","signature_id":"525cd225df12328345be68dc36b495d1d5100f3612e439dde87762dd1a04d4ca","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:135]: Unexpected error:\n \u003c*exported.ResponseError | 0xc0015da540\u003e: \n GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5b3f846-6506-4eda-9015-7af19755c284\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5b3f846-6506-4eda-9015-7af19755c284\",\n \"name\": \"c5b3f846-6506-4eda-9015-7af19755c284\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T13:10:10.437965568Z\",\n \"endTime\": \"2026-03-16T13:10:54.695097893Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-61/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-61/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"12303f11-8b3e-4d5f-affd-650a453fcf08\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 13:20:24 GMT\",\n ],\n \"Content-Length\": [\"1573\"],\n \"X-Envoy-Upstream-Service-Time\": [\"9\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000784660\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 99, 53, 98, 51, 102, 56, 52, 54, 45, 54, 53, 48, 54, 45, 52, 101, 100, 97, 45, 57, 48, 49, 53, 45, 55, 97, 102, 49, 57, 55, 53, 53, 99, 50, 56, 52, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 99, 53, 98, 51, 102, 56, 52, 54, 45, 54, 53, 48, 54, 45, 52, 101, 100, 97, 45, 57, 48, 49, 53, 45, 55, 97, 102, 49, 57, 55, 53, 53, 99, 50, 56, 52, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 116, 117, 115, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 114, 116, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 54, 84, 49, 51, 58, 49, 48, 58, 49, 48, 46, 52, 51, 55, 57, 54, 53, 53, 54, 56, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 110, 100, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 54, 84, 49, 51, 58, 49, 48, 58, 53, 52, 46, 54, 57, 53, 48, 57, 55, 56, 57, 51, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 102, 97, 105, 108, 101, 100, 32, 116, 111, 32, 103, 101, 116, 32, 109, 97, 110, 97, 103, 101, 100, 32, 105, 100, 101, 110, 116, 105, 116, 121, 32, 39, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 97, 114, 111, 45, 104, 99, 112, 45, 116, 101, 115, 116, 45, 109, 115, 105, 45, 99, 111, 110, 116, 97, 105, 110, 101, 114, 115, 45, 100, 101, 118, 45, 54, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 77, 97, 110, 97, 103, 101, 100, 73, 100, 101, 110, 116, 105, 116, 121, 47, 117, 115, 101, 114, 65, 115, 115, 105, 103, 110, 101, 100, 73, 100, 101, 110, 116, 105, 116, 105, 101, 115, 47, 99, 108, 111, 117, 100, 45, 110, 101, 116, 119, 111, 114, 107, 45, 99, 111, 110, 102, 105, 103, 39, 58, 32, 71, 69, 84, 32, 104, 116, 116, 112, 115, 58, 47, 47, 109, 97, 110, 97, 103, 101, 109, 101, 110, 116, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:135]: Unexpected error:\n \u003c*exported.ResponseError | 0xc0015da540\u003e: \n GET https://rp.j8596480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5b3f846-6506-4eda-9015-7af19755c284\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5b3f846-6506-4eda-9015-7af19755c284\",\n \"name\": \"c5b3f846-6506-4eda-9015-7af19755c284\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T13:10:10.437965568Z\",\n \"endTime\": \"2026-03-16T13:10:54.695097893Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-61/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-61/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"12303f11-8b3e-4d5f-affd-650a453fcf08\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 13:20:24 GMT\",\n ],\n \"Content-Length\": [\"1573\"],\n \"X-Envoy-Upstream-Service-Time\": [\"9\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000784660\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 99, 53, 98, 51, 102, 56, 52, 54, 45, 54, 53, 48, 54, 45, 52, 101, 100, 97, 45, 57, 48, 49, 53, 45, 55, 97, 102, 49, 57, 55, 53, 53, 99, 50, 56, 52, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 99, 53, 98, 51, 102, 56, 52, 54, 45, 54, 53, 48, 54, 45, 52, 101, 100, 97, 45, 57, 48, 49, 53, 45, 55, 97, 102, 49, 57, 55, 53, 53, 99, 50, 56, 52, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 116, 117, 115, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 114, 116, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 54, 84, 49, 51, 58, 49, 48, 58, 49, 48, 46, 52, 51, 55, 57, 54, 53, 53, 54, 56, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 110, 100, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 54, 84, 49, 51, 58, 49, 48, 58, 53, 52, 46, 54, 57, 53, 48, 57, 55, 56, 57, 51, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 102, 97, 105, 108, 101, 100, 32, 116, 111, 32, 103, 101, 116, 32, 109, 97, 110, 97, 103, 101, 100, 32, 105, 100, 101, 110, 116, 105, 116, 121, 32, 39, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 97, 114, 111, 45, 104, 99, 112, 45, 116, 101, 115, 116, 45, 109, 115, 105, 45, 99, 111, 110, 116, 97, 105, 110, 101, 114, 115, 45, 100, 101, 118, 45, 54, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 77, 97, 110, 97, 103, 101, 100, 73, 100, 101, 110, 116, 105, 116, 121, 47, 117, 115, 101, 114, 65, 115, 115, 105, 103, 110, 101, 100, 73, 100, 101, 110, 116, 105, 116, 105, 101, 115, 47, 99, 108, 111, 117, 100, 45, 110, 101, 116, 119, 111, 114, 107, 45, 99, 111, 110, 102, 105, 103, 39, 58, 32, 71, 69, 84, 32, 104, 116, 116, 112, 115, 58, 47, 47, 109, 97, 110, 97, 103, 101, 109, 101, 110, 116, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["0aa9ed5d94e7167a12aa30282eff19539fc3fc473772cb96b223d420df66775f","503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4453/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508109908971520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4453,"pr_state":"closed","pr_sha":"74066469c7ba25be3efcb17e4e6a47885ab78793","final_merged_sha":"74066469c7ba25be3efcb17e4e6a47885ab78793","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T11:38:08Z"},"failed_at":["e2e"],"failed_test_count":11,"occurrences":[{"row_id":"037ca6fdc0cce20b4858823680a3448feaa76bbb77eb5a619bd0669cdd6f91ae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4453/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508109908971520","occurred_at":"2026-03-16T11:38:08Z","failed_at":"e2e","signature_id":"de4a15bc46044618e4842b15219fdd4fadd939a9352a5d775bd31c79726587bf","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000c420c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listpwbm95\" in resourcegroup=\"ea-list-z65xhn\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000aa1440\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listpwbm95\\\" in resourcegroup=\\\"ea-list-z65xhn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000aa1420\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listpwbm95\\\" in resourcegroup=\\\"ea-list-z65xhn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000aa1400\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listpwbm95\\\" in resourcegroup=\\\"ea-list-z65xhn\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000c420c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listpwbm95\" in resourcegroup=\"ea-list-z65xhn\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000aa1440\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listpwbm95\\\" in resourcegroup=\\\"ea-list-z65xhn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000aa1420\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listpwbm95\\\" in resourcegroup=\\\"ea-list-z65xhn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000aa1400\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listpwbm95\\\" in resourcegroup=\\\"ea-list-z65xhn\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"167b71aefe25ea8361e05eb27ab9e1382bb9525dc1fb6b1de8465da9248d165a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4453/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508109908971520","occurred_at":"2026-03-16T11:38:08Z","failed_at":"e2e","signature_id":"ac98357c2d50a14e6745e8bf7f29725f35a9a11c372cc8fcec1ef206c11a7cd2","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e16240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-qm5msk\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007d5f20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-qm5msk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007d5f00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-qm5msk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007d5ee0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-qm5msk\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e16240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-qm5msk\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007d5f20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-qm5msk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007d5f00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-qm5msk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007d5ee0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-qm5msk\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"354f3f5ad694f31dd680d27cdc03be09747ce6804bfb03964050d02e5ede4721","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4453/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508109908971520","occurred_at":"2026-03-16T11:38:08Z","failed_at":"e2e","signature_id":"3ffc25aa484890b63632e1702d4389ff1b2d0fb67940f3d355fa9ca07418535f","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010a0378\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-5xj8sr\" in resourcegroup=\"rg-zstream-upgrade-4-20-77qhqq\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00104cfa0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-5xj8sr\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-77qhqq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00104cf80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-5xj8sr\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-77qhqq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00104cf60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-20-5xj8sr\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-77qhqq\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010a0378\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-5xj8sr\" in resourcegroup=\"rg-zstream-upgrade-4-20-77qhqq\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00104cfa0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-5xj8sr\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-77qhqq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00104cf80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-5xj8sr\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-77qhqq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00104cf60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-20-5xj8sr\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-77qhqq\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"426367cfaf6267560d58ecc984055e00b3b061c6d8f3c53596950d5673b4deba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4453/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508109908971520","occurred_at":"2026-03-16T11:38:08Z","failed_at":"e2e","signature_id":"7ce2087b9183e8070b40fbfca9ae7b5688b4d613fdbc730a7f89965312819406","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000012138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-lxsgpq\" in resourcegroup=\"admin-credential-lifecycle-test-57qnvt\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000980ca0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-lxsgpq\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-57qnvt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000980c80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-lxsgpq\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-57qnvt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000980c60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-lxsgpq\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-57qnvt\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000012138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-lxsgpq\" in resourcegroup=\"admin-credential-lifecycle-test-57qnvt\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000980ca0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-lxsgpq\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-57qnvt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000980c80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-lxsgpq\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-57qnvt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000980c60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-lxsgpq\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-57qnvt\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"719c8356c147cd8e6b42fa83e6df47290476000ab3d87599b1bb6fc7b0a3cd38","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4453/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508109908971520","occurred_at":"2026-03-16T11:38:08Z","failed_at":"e2e","signature_id":"a65129765d8892504fde13c700be694a7a3933bad3042b7eef145afe19235d11","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f2e228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-m9tz4j\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007acf80\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-m9tz4j\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007acf60\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-m9tz4j\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007acf40\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-m9tz4j\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f2e228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-m9tz4j\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007acf80\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-m9tz4j\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007acf60\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-m9tz4j\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007acf40\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-m9tz4j\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"7e40a856ba26ab33d529a6320b27b5a858a4f2d512b255011b4aff72aaa4a960","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4453/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508109908971520","occurred_at":"2026-03-16T11:38:08Z","failed_at":"e2e","signature_id":"145aa768fd3280e4587bc0e952592d6fb6f526d88179a69829df233091f7819d","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000fdbf50\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-qzw5x5\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007365a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-qzw5x5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000736580\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-qzw5x5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000736560\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-qzw5x5\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000fdbf50\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-qzw5x5\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007365a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-qzw5x5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000736580\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-qzw5x5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000736560\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-qzw5x5\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"8e65cde5f8684b266d50440931929afc5c239410babcfb5a0cfa7867618d3c30","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4453/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508109908971520","occurred_at":"2026-03-16T11:38:08Z","failed_at":"e2e","signature_id":"0f239336751f962bdb6851c6aa09362458308596f56333e76f96a2e7325ac385","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001112a40\u003e: \n failed to create HCP cluster cluster-zstream-4-19-qmv88n: failed waiting for cluster=\"cluster-zstream-4-19-qmv88n\" in resourcegroup=\"rg-zstream-upgrade-4-19-6vfmfp\" to finish creating: GET https://rp.j8971520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cffe6b5d-2234-4df7-a21f-204b7b5db928\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cffe6b5d-2234-4df7-a21f-204b7b5db928\",\n \"name\": \"cffe6b5d-2234-4df7-a21f-204b7b5db928\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:56:50.633649849Z\",\n \"endTime\": \"2026-03-16T14:57:37.094827871Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-69/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-69/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-qmv88n: failed waiting for cluster=\\\"cluster-zstream-4-19-qmv88n\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-6vfmfp\\\" to finish creating: GET https://rp.j8971520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cffe6b5d-2234-4df7-a21f-204b7b5db928\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cffe6b5d-2234-4df7-a21f-204b7b5db928\\\",\\n \\\"name\\\": \\\"cffe6b5d-2234-4df7-a21f-204b7b5db928\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:56:50.633649849Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:57:37.094827871Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-69/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-69/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001112a20\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-19-qmv88n\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-6vfmfp\\\" to finish creating: GET https://rp.j8971520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cffe6b5d-2234-4df7-a21f-204b7b5db928\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cffe6b5d-2234-4df7-a21f-204b7b5db928\\\",\\n \\\"name\\\": \\\"cffe6b5d-2234-4df7-a21f-204b7b5db928\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:56:50.633649849Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:57:37.094827871Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-69/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-69/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001112a40\u003e: \n failed to create HCP cluster cluster-zstream-4-19-qmv88n: failed waiting for cluster=\"cluster-zstream-4-19-qmv88n\" in resourcegroup=\"rg-zstream-upgrade-4-19-6vfmfp\" to finish creating: GET https://rp.j8971520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cffe6b5d-2234-4df7-a21f-204b7b5db928\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cffe6b5d-2234-4df7-a21f-204b7b5db928\",\n \"name\": \"cffe6b5d-2234-4df7-a21f-204b7b5db928\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:56:50.633649849Z\",\n \"endTime\": \"2026-03-16T14:57:37.094827871Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-69/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-69/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-qmv88n: failed waiting for cluster=\\\"cluster-zstream-4-19-qmv88n\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-6vfmfp\\\" to finish creating: GET https://rp.j8971520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cffe6b5d-2234-4df7-a21f-204b7b5db928\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cffe6b5d-2234-4df7-a21f-204b7b5db928\\\",\\n \\\"name\\\": \\\"cffe6b5d-2234-4df7-a21f-204b7b5db928\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:56:50.633649849Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:57:37.094827871Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-69/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-69/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001112a20\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-19-qmv88n\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-6vfmfp\\\" to finish creating: GET https://rp.j8971520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cffe6b5d-2234-4df7-a21f-204b7b5db928\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cffe6b5d-2234-4df7-a21f-204b7b5db928\\\",\\n \\\"name\\\": \\\"cffe6b5d-2234-4df7-a21f-204b7b5db928\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:56:50.633649849Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:57:37.094827871Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-69/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-69/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"aa81362f40b7864244060a02b234a6f453f7e628da97f444623df7b1742ce41b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4453/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508109908971520","occurred_at":"2026-03-16T11:38:08Z","failed_at":"e2e","signature_id":"966e935d6dfafed99b05e317b1c20dacc2146ee07fd90fff25760e098e9c147b","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e22300\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-d5qf5c\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0009237a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-d5qf5c\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000923600\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-d5qf5c\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009235e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-d5qf5c\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e22300\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-d5qf5c\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0009237a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-d5qf5c\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000923600\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-d5qf5c\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009235e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-d5qf5c\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"c97f912f4a7df1cedf7a52c441f57f662c04b01fa1ef1679b732cba55796ca73","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4453/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508109908971520","occurred_at":"2026-03-16T11:38:08Z","failed_at":"e2e","signature_id":"1cb7e15faae9980a41aa1c71b3bfa5d51634d92f255a8e1acf357f1391417e8a","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000fec390\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-2m2fvl\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a33380\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-2m2fvl\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a33360\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-2m2fvl\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a33340\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-2m2fvl\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000fec390\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-2m2fvl\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a33380\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-2m2fvl\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a33360\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-2m2fvl\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a33340\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-2m2fvl\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"db496c66b7d99950039c697e6fed2dcb3863b683e1f137b5c0358f0a6c07f647","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4453/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508109908971520","occurred_at":"2026-03-16T11:38:08Z","failed_at":"e2e","signature_id":"22d69a0f3a1d860ef3d6376fef436ac1019d2e7adb5a2200c96af56415572ef0","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:151]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b6e260\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster3-bxsgwq\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster3-bxsgwq/operationStatuses/08584279348715665067\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-m97f8n/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-igpwm4vimfr3y\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-basic-hcp-cluster3-bxsgwq\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster3-bxsgwq/operationStatuses/08584279348715665067\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-m97f8n/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-igpwm4vimfr3y\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000b6e240\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-basic-hcp-cluster3-bxsgwq\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster3-bxsgwq/operationStatuses/08584279348715665067\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-m97f8n/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-igpwm4vimfr3y\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Ple...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:151]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b6e260\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster3-bxsgwq\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster3-bxsgwq/operationStatuses/08584279348715665067\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-m97f8n/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-igpwm4vimfr3y\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-basic-hcp-cluster3-bxsgwq\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster3-bxsgwq/operationStatuses/08584279348715665067\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-m97f8n/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-igpwm4vimfr3y\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000b6e240\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-basic-hcp-cluster3-bxsgwq\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster3-bxsgwq/operationStatuses/08584279348715665067\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-m97f8n/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-igpwm4vimfr3y\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Ple...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"e4c5e524fe71054e517c6a3b2fec53c4f6c17dc78c83c2b7857226b7993b87d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4453/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508109908971520","occurred_at":"2026-03-16T11:38:08Z","failed_at":"e2e","signature_id":"8301fe11d9c019bf7c4a79dfa863cb9bf7f5c44416075c3352f89234552782eb","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00093a7e0\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-627pgj\" to finish creating: GET https://rp.j8971520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7084ccd4-bfdb-480b-8ef1-2dcb5788cdcb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7084ccd4-bfdb-480b-8ef1-2dcb5788cdcb\",\n \"name\": \"7084ccd4-bfdb-480b-8ef1-2dcb5788cdcb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:32:49.565417077Z\",\n \"endTime\": \"2026-03-16T14:33:53.85749565Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-132/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-132/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-627pgj\\\" to finish creating: GET https://rp.j8971520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7084ccd4-bfdb-480b-8ef1-2dcb5788cdcb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7084ccd4-bfdb-480b-8ef1-2dcb5788cdcb\\\",\\n \\\"name\\\": \\\"7084ccd4-bfdb-480b-8ef1-2dcb5788cdcb\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:32:49.565417077Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:33:53.85749565Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-132/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-132/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00093a7c0\u003e{\n msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-627pgj\\\" to finish creating: GET https://rp.j8971520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7084ccd4-bfdb-480b-8ef1-2dcb5788cdcb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7084ccd4-bfdb-480b-8ef1-2dcb5788cdcb\\\",\\n \\\"name\\\": \\\"7084ccd4-bfdb-480b-8ef1-2dcb5788cdcb\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:32:49.565417077Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:33:53.85749565Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-132/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-132/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00093a7e0\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-627pgj\" to finish creating: GET https://rp.j8971520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7084ccd4-bfdb-480b-8ef1-2dcb5788cdcb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7084ccd4-bfdb-480b-8ef1-2dcb5788cdcb\",\n \"name\": \"7084ccd4-bfdb-480b-8ef1-2dcb5788cdcb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:32:49.565417077Z\",\n \"endTime\": \"2026-03-16T14:33:53.85749565Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-132/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-132/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-627pgj\\\" to finish creating: GET https://rp.j8971520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7084ccd4-bfdb-480b-8ef1-2dcb5788cdcb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7084ccd4-bfdb-480b-8ef1-2dcb5788cdcb\\\",\\n \\\"name\\\": \\\"7084ccd4-bfdb-480b-8ef1-2dcb5788cdcb\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:32:49.565417077Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:33:53.85749565Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-132/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-132/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00093a7c0\u003e{\n msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-627pgj\\\" to finish creating: GET https://rp.j8971520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7084ccd4-bfdb-480b-8ef1-2dcb5788cdcb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7084ccd4-bfdb-480b-8ef1-2dcb5788cdcb\\\",\\n \\\"name\\\": \\\"7084ccd4-bfdb-480b-8ef1-2dcb5788cdcb\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:32:49.565417077Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:33:53.85749565Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-132/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-132/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":11,"failure_pattern_ids":["503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d"],"matched_occurrences":11,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508077847711744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4454,"pr_state":"closed","pr_sha":"38f866ad42f9613c0978385116cd5b0c6df796c1","final_merged_sha":"38f866ad42f9613c0978385116cd5b0c6df796c1","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T11:38:01Z"},"failed_at":["e2e"],"failed_test_count":20,"occurrences":[{"row_id":"06c90f1d0b9ed4dafb3352970dd4748f0dbdf72bbdced8d2430e3130d6b3a3b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508077847711744","occurred_at":"2026-03-16T11:38:01Z","failed_at":"e2e","signature_id":"e4495486efaf54db5f6623c6b1a185f811aea7afe833d23e81927f84bdcb4492","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d84258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listptm4dr\" in resourcegroup=\"ea-list-wml2kk\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000cf6280\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listptm4dr\\\" in resourcegroup=\\\"ea-list-wml2kk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000cf6240\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listptm4dr\\\" in resourcegroup=\\\"ea-list-wml2kk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000cf6220\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listptm4dr\\\" in resourcegroup=\\\"ea-list-wml2kk\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d84258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listptm4dr\" in resourcegroup=\"ea-list-wml2kk\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000cf6280\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listptm4dr\\\" in resourcegroup=\\\"ea-list-wml2kk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000cf6240\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listptm4dr\\\" in resourcegroup=\\\"ea-list-wml2kk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000cf6220\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listptm4dr\\\" in resourcegroup=\\\"ea-list-wml2kk\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"104989b4d0cd8296011460fb0435a4bacaa6533b4977e176c498145b70bfac26","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508077847711744","occurred_at":"2026-03-16T11:38:01Z","failed_at":"e2e","signature_id":"9b9c1947df8b07b42636bceaf141d4d7ccecd0d863463f59e0fe3ea5caf3c760","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:242]: Unexpected error:\n \u003c*errors.joinError | 0xc000ffeed0\u003e: \n context deadline exceeded\n failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\n {\n errs: [\n \u003ccontext.deadlineExceededError\u003e{},\n \u003c*fmt.wrapError | 0xc000fe65e0\u003e{\n msg: \"failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xc000fe65c0\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:242]: Unexpected error:\n \u003c*errors.joinError | 0xc000ffeed0\u003e: \n context deadline exceeded\n failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\n {\n errs: [\n \u003ccontext.deadlineExceededError\u003e{},\n \u003c*fmt.wrapError | 0xc000fe65e0\u003e{\n msg: \"failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xc000fe65c0\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"27bf43830385464e60fff2411ebc0a00bf4524da8cd0071a65cda3afa5965e2f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508077847711744","occurred_at":"2026-03-16T11:38:01Z","failed_at":"e2e","signature_id":"f8480b05403db8c9ac06d4121e163b4bae899dd0dbcb855e5923be8a7eb78f10","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0008dc840\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-20-t27s5z-d2k5jf\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-t27s5z-d2k5jf/operationStatuses/08584279347964614527\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-p2xqvm/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-xruflpgmwocnq\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-55/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-20-t27s5z-d2k5jf\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-t27s5z-d2k5jf/operationStatuses/08584279347964614527\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-p2xqvm/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-xruflpgmwocnq\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-55/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0008dc7e0\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-20-t27s5z-d2k5jf\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-t27s5z-d2k5jf/operationStatuses/08584279347964614527\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nER...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0008dc840\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-20-t27s5z-d2k5jf\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-t27s5z-d2k5jf/operationStatuses/08584279347964614527\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-p2xqvm/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-xruflpgmwocnq\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-55/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-20-t27s5z-d2k5jf\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-t27s5z-d2k5jf/operationStatuses/08584279347964614527\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-p2xqvm/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-xruflpgmwocnq\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-55/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0008dc7e0\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-20-t27s5z-d2k5jf\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-t27s5z-d2k5jf/operationStatuses/08584279347964614527\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nER...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"2c28c7be333e43dd099afee428e0ab9017263fb9e597fdad856661948a009096","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508077847711744","occurred_at":"2026-03-16T11:38:01Z","failed_at":"e2e","signature_id":"7b551f8c5b25ae0ae5926ccb40a47f8fde595433c6c76bf89e419786222ed47d","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010c6258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-8mzj6g\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000ea2e20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-8mzj6g\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ea2e00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-8mzj6g\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ea2de0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-8mzj6g\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010c6258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-8mzj6g\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000ea2e20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-8mzj6g\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ea2e00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-8mzj6g\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ea2de0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-8mzj6g\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"37a615ccaf83bb645b3c028f9586c3d0ae7f6d8e506392a87ce5b84db1db8e5f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508077847711744","occurred_at":"2026-03-16T11:38:01Z","failed_at":"e2e","signature_id":"8c6182530eb21b3466e4ebbde2e29645db0914df70f2ca962a115d13887ca917","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000777540\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-19-nsdxc6-fqc8ch\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-19-nsdxc6-fqc8ch/operationStatuses/08584279348074294009\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-9dgkn9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-d22ejdsr2z4k6\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-114/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-19-nsdxc6-fqc8ch\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-19-nsdxc6-fqc8ch/operationStatuses/08584279348074294009\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-9dgkn9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-d22ejdsr2z4k6\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000777540\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-19-nsdxc6-fqc8ch\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-19-nsdxc6-fqc8ch/operationStatuses/08584279348074294009\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-9dgkn9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-d22ejdsr2z4k6\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-114/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-19-nsdxc6-fqc8ch\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-19-nsdxc6-fqc8ch/operationStatuses/08584279348074294009\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-9dgkn9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-d22ejdsr2z4k6\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"422106092943441dc097f38a3d33642d7b74a9d62e5fad466abb72339e573e18","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508077847711744","occurred_at":"2026-03-16T11:38:01Z","failed_at":"e2e","signature_id":"2aea92acbc634c55b66f4e2f0dbf00b6a0c1aa14b5f8668db5489b4413ded607","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:79]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000bb19c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-negative-tests-cluster-wgq4wx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-negative-tests-cluster-wgq4wx/operationStatuses/08584279346171694286\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-skmwld/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oeayaalhj7wjy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-114/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-negative-tests-cluster-wgq4wx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-negative-tests-cluster-wgq4wx/operationStatuses/08584279346171694286\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-skmwld/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oeayaalhj7wjy\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:79]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000bb19c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-negative-tests-cluster-wgq4wx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-negative-tests-cluster-wgq4wx/operationStatuses/08584279346171694286\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-skmwld/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oeayaalhj7wjy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-114/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-negative-tests-cluster-wgq4wx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-negative-tests-cluster-wgq4wx/operationStatuses/08584279346171694286\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-skmwld/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oeayaalhj7wjy\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"55cad931ca193f745c4e2deda7ff1fbccb089e54d2e6681128ec061e4b98ede4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508077847711744","occurred_at":"2026-03-16T11:38:01Z","failed_at":"e2e","signature_id":"bf5ad3f92cefa28a4a6d1eea8350de7684680f260d2b18cb65105b2d63eec411","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0001da140\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-kl2nls\" to finish creating: GET https://rp.j7711744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc8ee5d3-6bb5-4055-8f4b-9e79f49627cc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc8ee5d3-6bb5-4055-8f4b-9e79f49627cc\",\n \"name\": \"fc8ee5d3-6bb5-4055-8f4b-9e79f49627cc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:36:47.922929088Z\",\n \"endTime\": \"2026-03-16T14:37:44.479957319Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-kl2nls\\\" to finish creating: GET https://rp.j7711744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc8ee5d3-6bb5-4055-8f4b-9e79f49627cc\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc8ee5d3-6bb5-4055-8f4b-9e79f49627cc\\\",\\n \\\"name\\\": \\\"fc8ee5d3-6bb5-4055-8f4b-9e79f49627cc\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:36:47.922929088Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:37:44.479957319Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0001da120\u003e{\n msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-kl2nls\\\" to finish creating: GET https://rp.j7711744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc8ee5d3-6bb5-4055-8f4b-9e79f49627cc\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc8ee5d3-6bb5-4055-8f4b-9e79f49627cc\\\",\\n \\\"name\\\": \\\"fc8ee5d3-6bb5-4055-8f4b-9e79f49627cc\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:36:47.922929088Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:37:44.479957319Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n----------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0001da140\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-kl2nls\" to finish creating: GET https://rp.j7711744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc8ee5d3-6bb5-4055-8f4b-9e79f49627cc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc8ee5d3-6bb5-4055-8f4b-9e79f49627cc\",\n \"name\": \"fc8ee5d3-6bb5-4055-8f4b-9e79f49627cc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:36:47.922929088Z\",\n \"endTime\": \"2026-03-16T14:37:44.479957319Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-kl2nls\\\" to finish creating: GET https://rp.j7711744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc8ee5d3-6bb5-4055-8f4b-9e79f49627cc\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc8ee5d3-6bb5-4055-8f4b-9e79f49627cc\\\",\\n \\\"name\\\": \\\"fc8ee5d3-6bb5-4055-8f4b-9e79f49627cc\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:36:47.922929088Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:37:44.479957319Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0001da120\u003e{\n msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-kl2nls\\\" to finish creating: GET https://rp.j7711744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc8ee5d3-6bb5-4055-8f4b-9e79f49627cc\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc8ee5d3-6bb5-4055-8f4b-9e79f49627cc\\\",\\n \\\"name\\\": \\\"fc8ee5d3-6bb5-4055-8f4b-9e79f49627cc\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:36:47.922929088Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:37:44.479957319Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n----------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5d5e972e08c29d16b32698bd51e3369ce40e04dde34fa805072ef92e178b84a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508077847711744","occurred_at":"2026-03-16T11:38:01Z","failed_at":"e2e","signature_id":"e7b569273a42a7ec5d8831ba059322bdf04aa5d3c17fae39da9229b3a4834c8b","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00157a1c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-mq9qzt\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00068cb40\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-mq9qzt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00068cb20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-mq9qzt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00068cb00\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-mq9qzt\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00157a1c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-mq9qzt\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00068cb40\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-mq9qzt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00068cb20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-mq9qzt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00068cb00\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-mq9qzt\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"5d7e670313bd7f93718581d9eb3059738de38709e498403bfdafe9a91728cbd6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508077847711744","occurred_at":"2026-03-16T11:38:01Z","failed_at":"e2e","signature_id":"02ec88b414da3aa362e088b0e877ec6ddb54b7b9775092194bcab3836503bece","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0011f82e8\u003e: \n failed to cleanup resource group: failed waiting for resourcegroup=\"e2e-cidr-connectivity-wddhhw\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000989600\u003e{\n msg: \"failed to cleanup resource group: failed waiting for resourcegroup=\\\"e2e-cidr-connectivity-wddhhw\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009895e0\u003e{\n msg: \"failed waiting for resourcegroup=\\\"e2e-cidr-connectivity-wddhhw\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0011f82e8\u003e: \n failed to cleanup resource group: failed waiting for resourcegroup=\"e2e-cidr-connectivity-wddhhw\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000989600\u003e{\n msg: \"failed to cleanup resource group: failed waiting for resourcegroup=\\\"e2e-cidr-connectivity-wddhhw\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009895e0\u003e{\n msg: \"failed waiting for resourcegroup=\\\"e2e-cidr-connectivity-wddhhw\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d55c4a29bbbf19a43335bfa2157018ebc82a1c8fe8cbea22f78ef2ca4c570f1a","failure_pattern":"failed to cleanup resource group: failed waiting for resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: failed waiting for resourcegroup=\"autoscaling-cluster-2bvkgc\" to finish deleting: context canceled"}},{"row_id":"82732dd833d45f612660b89945adc225b2986e84ff7736a89ac889dd0b69b734","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508077847711744","occurred_at":"2026-03-16T11:38:01Z","failed_at":"e2e","signature_id":"25cd762c6975f97f7cb585c9c0ae9840857126c5e3045403ceaa7f197e8c5b8e","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0013c01e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-rdwmn6\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000832d60\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-rdwmn6\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000832d40\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-rdwmn6\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000832d20\u003e{\n msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-rdwmn6\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0013c01e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-rdwmn6\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000832d60\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-rdwmn6\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000832d40\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-rdwmn6\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000832d20\u003e{\n msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-rdwmn6\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"947b50fdc68b1a1a068a46250dff7b1054b426f48ebcaf44faf30b3ce208eba1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508077847711744","occurred_at":"2026-03-16T11:38:01Z","failed_at":"e2e","signature_id":"3836292eb3a8ef099cb16c86de45c811eedf05580ce9076176fff089574a18df","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000dd8678\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-7m89cx\" in resourcegroup=\"rg-zstream-upgrade-4-21-rf6mkp\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000c9f520\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-7m89cx\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-rf6mkp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000c9f500\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-7m89cx\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-rf6mkp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000c9f4e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-7m89cx\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-rf6mkp\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000dd8678\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-7m89cx\" in resourcegroup=\"rg-zstream-upgrade-4-21-rf6mkp\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000c9f520\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-7m89cx\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-rf6mkp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000c9f500\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-7m89cx\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-rf6mkp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000c9f4e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-7m89cx\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-rf6mkp\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"b36c93f6ca1226a3428606c78e9c293826ded36b5e052d863df81fb72e7252ba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508077847711744","occurred_at":"2026-03-16T11:38:01Z","failed_at":"e2e","signature_id":"26fdb09925284ebd42000340721c04501a0bbf474af82212bae80209ce5703fa","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001526240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-scqfhl\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007b3920\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-scqfhl\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007b3900\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-scqfhl\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007b38e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-scqfhl\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001526240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-scqfhl\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007b3920\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-scqfhl\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007b3900\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-scqfhl\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007b38e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-scqfhl\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"b46fb2d7fa6bdf9fdc5b6ba3cd7f6ac0c8d6d595a9dfc6481e2ec57ab78ab1ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508077847711744","occurred_at":"2026-03-16T11:38:01Z","failed_at":"e2e","signature_id":"f31326fe2ea02ecce15f263d0c017116d6e9116951b9d447185ee4b56307df0f","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d94378\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-fczs4p\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0006fd920\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-fczs4p\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006fd8e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-fczs4p\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006fd8c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-fczs4p\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d94378\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-fczs4p\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0006fd920\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-fczs4p\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006fd8e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-fczs4p\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006fd8c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-fczs4p\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"b5ae06c53a0d26f8d8a7a8b1b78c994017e4c45dd20a82b3ef6ce33c91ec7673","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508077847711744","occurred_at":"2026-03-16T11:38:01Z","failed_at":"e2e","signature_id":"cbdb4f5340d48d0a1a1745402c656f6973dc91953b22f4749685172221294107","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e4c600\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-bjhgrb\" to finish creating: GET https://rp.j7711744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb56abe4-c630-47aa-9f92-e717ef68f70b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb56abe4-c630-47aa-9f92-e717ef68f70b\",\n \"name\": \"cb56abe4-c630-47aa-9f92-e717ef68f70b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:37:42.088418326Z\",\n \"endTime\": \"2026-03-16T14:38:25.667213121Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-114/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-114/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-bjhgrb\\\" to finish creating: GET https://rp.j7711744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb56abe4-c630-47aa-9f92-e717ef68f70b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb56abe4-c630-47aa-9f92-e717ef68f70b\\\",\\n \\\"name\\\": \\\"cb56abe4-c630-47aa-9f92-e717ef68f70b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:37:42.088418326Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:38:25.667213121Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-114/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-114/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e4c5e0\u003e{\n msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-bjhgrb\\\" to finish creating: GET https://rp.j7711744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb56abe4-c630-47aa-9f92-e717ef68f70b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb56abe4-c630-47aa-9f92-e717ef68f70b\\\",\\n \\\"name\\\": \\\"cb56abe4-c630-47aa-9f92-e717ef68f70b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:37:42.088418326Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:38:25.667213121Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-114/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-114/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n---------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e4c600\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-bjhgrb\" to finish creating: GET https://rp.j7711744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb56abe4-c630-47aa-9f92-e717ef68f70b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb56abe4-c630-47aa-9f92-e717ef68f70b\",\n \"name\": \"cb56abe4-c630-47aa-9f92-e717ef68f70b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:37:42.088418326Z\",\n \"endTime\": \"2026-03-16T14:38:25.667213121Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-114/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-114/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-bjhgrb\\\" to finish creating: GET https://rp.j7711744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb56abe4-c630-47aa-9f92-e717ef68f70b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb56abe4-c630-47aa-9f92-e717ef68f70b\\\",\\n \\\"name\\\": \\\"cb56abe4-c630-47aa-9f92-e717ef68f70b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:37:42.088418326Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:38:25.667213121Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-114/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-114/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e4c5e0\u003e{\n msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-bjhgrb\\\" to finish creating: GET https://rp.j7711744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb56abe4-c630-47aa-9f92-e717ef68f70b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb56abe4-c630-47aa-9f92-e717ef68f70b\\\",\\n \\\"name\\\": \\\"cb56abe4-c630-47aa-9f92-e717ef68f70b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:37:42.088418326Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:38:25.667213121Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-114/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-114/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n---------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b9831866fa240c5832d5e999758af47a400bbf50dae2556f26598527c7ab3a28","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508077847711744","occurred_at":"2026-03-16T11:38:01Z","failed_at":"e2e","signature_id":"a6138806d0fb2e6a3bd2eceb973d118f1e70b0d33c6f31c3b277446fc245a354","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e9e630\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-k688nh\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0014f5180\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-k688nh\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0014f5160\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-k688nh\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0014f5140\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-k688nh\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e9e630\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-k688nh\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0014f5180\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-k688nh\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0014f5160\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-k688nh\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0014f5140\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-k688nh\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"bb8a60b79ee1704dd49bc2c15d303b006cba72381e067ac4642ce380cfec8ca6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508077847711744","occurred_at":"2026-03-16T11:38:01Z","failed_at":"e2e","signature_id":"f44653127b179bf16e49b08d59913b1648c87f3053b922d1eedd49d87dc64c62","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000189140\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-nd6h7p\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000661d20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-nd6h7p\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000661d00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-nd6h7p\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000661ce0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-nd6h7p\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000189140\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-nd6h7p\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000661d20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-nd6h7p\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000661d00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-nd6h7p\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000661ce0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-nd6h7p\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"db4c24e2585a5a9491e977bad0c1cc597c8bd9cecca9ae0cc5f9136749c31c7a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508077847711744","occurred_at":"2026-03-16T11:38:01Z","failed_at":"e2e","signature_id":"564ea982fe0f6c51e37c35eb5a2c4551fa7791400bc9fe9a55c9a7bf5f6775a0","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:73]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00148c5a0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-update-nodes-hcp-cluster-9gt4qd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-9gt4qd/operationStatuses/08584279346169881662\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-55/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-7fn6kk/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-np-update-nodes-hcp-cluster-9gt4qd\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-9gt4qd/operationStatuses/08584279346169881662\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-55/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-7fn6kk/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:73]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00148c5a0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-update-nodes-hcp-cluster-9gt4qd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-9gt4qd/operationStatuses/08584279346169881662\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-55/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-7fn6kk/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-np-update-nodes-hcp-cluster-9gt4qd\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-9gt4qd/operationStatuses/08584279346169881662\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-55/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-7fn6kk/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"db957e10c2a9cccc5757764e58b890363305f94548546575498bcd95b919f331","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508077847711744","occurred_at":"2026-03-16T11:38:01Z","failed_at":"e2e","signature_id":"4072c33bc2b7adbe04e853877abe2c4201a0e9f5587a5c4f1bb9838fd48f16e1","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001494108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-cdsw9p\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00088f8e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-cdsw9p\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00088f8c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-cdsw9p\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00088f6e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-cdsw9p\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001494108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-cdsw9p\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00088f8e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-cdsw9p\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00088f8c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-cdsw9p\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00088f6e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-cdsw9p\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"e5b972b5de2d3608891dc48ef29de6588730a02dfeac537ffaad1f6649317768","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508077847711744","occurred_at":"2026-03-16T11:38:01Z","failed_at":"e2e","signature_id":"d8d14f2d5caf158e03b5dd60da4c32cfd478ede992fb597089b0921130a4761f","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001802270\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-msmfdt\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000eaef20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-msmfdt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000eaef00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-msmfdt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000eaeee0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-msmfdt\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001802270\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-msmfdt\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000eaef20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-msmfdt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000eaef00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-msmfdt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000eaeee0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-msmfdt\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"f3266685071cb5b51d83a8df9ec5739a108ebc5eaf026655e9c3e8b7bbc54685","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4454/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508077847711744","occurred_at":"2026-03-16T11:38:01Z","failed_at":"e2e","signature_id":"199df6731dfbe76f447a97177777033fd61617c44775a71689db1edbcc44852b","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000fd62e8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-cwwjhn\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000b87080\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-cwwjhn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b87060\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-cwwjhn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b87040\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-cwwjhn\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000fd62e8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-cwwjhn\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000b87080\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-cwwjhn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b87060\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-cwwjhn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b87040\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-cwwjhn\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":20,"failure_pattern_ids":["503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","d55c4a29bbbf19a43335bfa2157018ebc82a1c8fe8cbea22f78ef2ca4c570f1a"],"matched_occurrences":20,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4455/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508047766163456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4455,"pr_state":"closed","pr_sha":"a484243a8fad178e081f8e0f89d8db8e300a58a1","final_merged_sha":"7b9532af16d82b9a9786a1179527a9bf0cc955bb","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:37:53Z"},"failed_at":["e2e"],"failed_test_count":9,"occurrences":[{"row_id":"11a03673f52006919eca66cd037f6cdb6fc4b7a8cc1f26ea2a8ac524206404d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4455/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508047766163456","occurred_at":"2026-03-16T11:37:53Z","failed_at":"e2e","signature_id":"159b185065e5564bdbbd349c7432bd1244894b7bfa175dbaedf07bc260a3f3d5","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:89]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0000dfe20\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-fbwdrn\" to finish creating: GET https://rp.j6163456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/06c30845-f72f-4089-8e52-feb48ef60713\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/06c30845-f72f-4089-8e52-feb48ef60713\",\n \"name\": \"06c30845-f72f-4089-8e52-feb48ef60713\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:56:24.369399603Z\",\n \"endTime\": \"2026-03-16T12:57:38.788738032Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-173/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-173/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fbwdrn\\\" to finish creating: GET https://rp.j6163456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/06c30845-f72f-4089-8e52-feb48ef60713\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/06c30845-f72f-4089-8e52-feb48ef60713\\\",\\n \\\"name\\\": \\\"06c30845-f72f-4089-8e52-feb48ef60713\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:56:24.369399603Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:57:38.788738032Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-173/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-173/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0000dfe00\u003e{\n msg: \"failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fbwdrn\\\" to finish creating: GET https://rp.j6163456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/06c30845-f72f-4089-8e52-feb48ef60713\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/06c30845-f72f-4089-8e52-feb48ef60713\\\",\\n \\\"name\\\": \\\"06c30845-f72f-4089-8e52-feb48ef60713\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:56:24.369399603Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:57:38.788738032Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-173/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-173/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n----------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:89]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0000dfe20\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-fbwdrn\" to finish creating: GET https://rp.j6163456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/06c30845-f72f-4089-8e52-feb48ef60713\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/06c30845-f72f-4089-8e52-feb48ef60713\",\n \"name\": \"06c30845-f72f-4089-8e52-feb48ef60713\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:56:24.369399603Z\",\n \"endTime\": \"2026-03-16T12:57:38.788738032Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-173/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-173/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fbwdrn\\\" to finish creating: GET https://rp.j6163456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/06c30845-f72f-4089-8e52-feb48ef60713\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/06c30845-f72f-4089-8e52-feb48ef60713\\\",\\n \\\"name\\\": \\\"06c30845-f72f-4089-8e52-feb48ef60713\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:56:24.369399603Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:57:38.788738032Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-173/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-173/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0000dfe00\u003e{\n msg: \"failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fbwdrn\\\" to finish creating: GET https://rp.j6163456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/06c30845-f72f-4089-8e52-feb48ef60713\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/06c30845-f72f-4089-8e52-feb48ef60713\\\",\\n \\\"name\\\": \\\"06c30845-f72f-4089-8e52-feb48ef60713\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:56:24.369399603Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:57:38.788738032Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-173/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-173/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n----------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"648d55fd08ab80c957a307fc26a9ec22fb7522b50b74b02044baaec066d19270","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4455/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508047766163456","occurred_at":"2026-03-16T11:37:53Z","failed_at":"e2e","signature_id":"e0415ab5157c53be6277a14ba9253d35a8f49d28f48fb3e6f05f9f3e557c8943","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0016241c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-qjzfxw\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000783060\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-qjzfxw\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000783020\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-qjzfxw\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000783000\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-qjzfxw\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0016241c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-qjzfxw\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000783060\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-qjzfxw\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000783020\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-qjzfxw\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000783000\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-qjzfxw\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"75eeeb58bdca1c481ec14ad8493fde3ecc09abf1ee83e0e106127e7b0960ff51","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4455/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508047766163456","occurred_at":"2026-03-16T11:37:53Z","failed_at":"e2e","signature_id":"520a2c1b5840749250b667fd29de20911b111a64854c1476fa2bc39f8a68d03a","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ecc348\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-5bxvxl\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0006f7b80\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-5bxvxl\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006f7b60\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-5bxvxl\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006f7b40\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-5bxvxl\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ecc348\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-5bxvxl\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0006f7b80\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-5bxvxl\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006f7b60\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-5bxvxl\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006f7b40\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-5bxvxl\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"88faab3aa9df8cc45fdb8d29de2ac513a263b3e40fe48c450a1941ddf91f3e3a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4455/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508047766163456","occurred_at":"2026-03-16T11:37:53Z","failed_at":"e2e","signature_id":"83516bb567fbbfcd57d8827efe704323b2a7fcfec0257f73d4f1dcdd395fbcd3","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00136c258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-x8tkj7\" in resourcegroup=\"rg-zstream-upgrade-4-20-5zgp48\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000ebf1c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-x8tkj7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-5zgp48\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ebf1a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-x8tkj7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-5zgp48\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ebf180\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-20-x8tkj7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-5zgp48\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00136c258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-x8tkj7\" in resourcegroup=\"rg-zstream-upgrade-4-20-5zgp48\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000ebf1c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-x8tkj7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-5zgp48\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ebf1a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-x8tkj7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-5zgp48\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ebf180\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-20-x8tkj7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-5zgp48\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"b4155d9b5e7ae85f311c8ed4aafda329fe29ac7afbbbd1bd4f5d99bb40a33237","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4455/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508047766163456","occurred_at":"2026-03-16T11:37:53Z","failed_at":"e2e","signature_id":"d059859af979bce32f974ca29bc2d08dc662618e5b36df4ff251a6345ab8f262","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d8e228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-6jwdk7\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00108b0a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-6jwdk7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00108b080\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-6jwdk7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00108b060\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-6jwdk7\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d8e228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-6jwdk7\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00108b0a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-6jwdk7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00108b080\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-6jwdk7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00108b060\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-6jwdk7\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"b4946b4ed9b034be8c8b861e2fab6322c169bd408fb8b1979bf30eb6ba4ddd4b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4455/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508047766163456","occurred_at":"2026-03-16T11:37:53Z","failed_at":"e2e","signature_id":"d2c42475bff3018cec621d58631a3e4308b3f168b3fbed0d3c9d34ac435ea14a","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000012f30\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-zpzmp2\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000ea0460\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-zpzmp2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ea0440\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-zpzmp2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ea0420\u003e{\n msg: \"failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-zpzmp2\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000012f30\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-zpzmp2\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000ea0460\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-zpzmp2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ea0440\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-zpzmp2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ea0420\u003e{\n msg: \"failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-zpzmp2\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"be20cb326329d9e9493e8329fd982013a066468eb1576f799459f7a184393cb2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4455/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508047766163456","occurred_at":"2026-03-16T11:37:53Z","failed_at":"e2e","signature_id":"96a43fe7695025101437d4f1605b7a1722eadd8f911b21f085a85805deefb03c","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d34468\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-8jfx2z\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000905a60\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-8jfx2z\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000905a40\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-8jfx2z\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000905a20\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-8jfx2z\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d34468\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-8jfx2z\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000905a60\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-8jfx2z\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000905a40\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-8jfx2z\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000905a20\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-8jfx2z\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"d0116733b24775bb3b4b1f2cdf284102b4f0c87048bf88feb4fe535ebc6b6a90","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4455/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508047766163456","occurred_at":"2026-03-16T11:37:53Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"e8ec4a8429313db34a0f5c63a63ab2549879798e32e6b95bae377296d51452a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4455/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508047766163456","occurred_at":"2026-03-16T11:37:53Z","failed_at":"e2e","signature_id":"dc21188e05cadf07685cca595ea0f6e50c047330a84442c66e3345777836de7a","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b894c0\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-7wtncf\" to finish creating: GET https://rp.j6163456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/07d751a2-253f-4673-8992-6459cd0efabc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/07d751a2-253f-4673-8992-6459cd0efabc\",\n \"name\": \"07d751a2-253f-4673-8992-6459cd0efabc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:32:29.510982837Z\",\n \"endTime\": \"2026-03-16T12:33:20.699146823Z\",\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"Cluster provisioning failed due to multiple errors\",\n \"details\": [\n {\n \"code\": \"InternalServerError\",\n \"message\": \"Failed to get virtual network 'engineering-vnet-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-7wtncf/providers/Microsoft.Network/virtualNetworks/engineering-vnet-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n },\n {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-7wtncf\\\" to finish creating: GET https://rp.j6163456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/07d751a2-253f-4673-8992-6459cd0efabc\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/07d751a2-253f-4673-8992-6459cd0efabc\\\",\\n \\\"name\\\": \\\"07d751a2-253f-4673-8992-6459cd0efabc\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:32:29.510982837Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:33:20.699146823Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'engineering-vnet-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-7wtncf/providers/Microsoft.Network/virtualNetworks/engineering-vnet-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n },\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000b894a0\u003e{\n msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-7wtncf\\\" to finish creating: GET https://rp.j6163456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/07d751a2-253f-4673-8992-6459cd0efabc\\n-----------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b894c0\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-7wtncf\" to finish creating: GET https://rp.j6163456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/07d751a2-253f-4673-8992-6459cd0efabc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/07d751a2-253f-4673-8992-6459cd0efabc\",\n \"name\": \"07d751a2-253f-4673-8992-6459cd0efabc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:32:29.510982837Z\",\n \"endTime\": \"2026-03-16T12:33:20.699146823Z\",\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"Cluster provisioning failed due to multiple errors\",\n \"details\": [\n {\n \"code\": \"InternalServerError\",\n \"message\": \"Failed to get virtual network 'engineering-vnet-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-7wtncf/providers/Microsoft.Network/virtualNetworks/engineering-vnet-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n },\n {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-7wtncf\\\" to finish creating: GET https://rp.j6163456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/07d751a2-253f-4673-8992-6459cd0efabc\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/07d751a2-253f-4673-8992-6459cd0efabc\\\",\\n \\\"name\\\": \\\"07d751a2-253f-4673-8992-6459cd0efabc\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:32:29.510982837Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:33:20.699146823Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'engineering-vnet-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-7wtncf/providers/Microsoft.Network/virtualNetworks/engineering-vnet-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n },\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000b894a0\u003e{\n msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-7wtncf\\\" to finish creating: GET https://rp.j6163456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/07d751a2-253f-4673-8992-6459cd0efabc\\n-----------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0aa9ed5d94e7167a12aa30282eff19539fc3fc473772cb96b223d420df66775f","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}}],"failure_pattern_summary":{"signature_count":9,"failure_pattern_ids":["0aa9ed5d94e7167a12aa30282eff19539fc3fc473772cb96b223d420df66775f","503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4"],"matched_occurrences":9,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508016086585344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4457,"pr_state":"open","pr_sha":"c6d34c6bc54bf82f40b7a2b9e91f43b977c11e31","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:37:46Z"},"failed_at":["e2e"],"failed_test_count":16,"occurrences":[{"row_id":"1c54e25dd50f8f5edf05ccd79b66e0d654d90846919648c0a8f40d6cf08c4db5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508016086585344","occurred_at":"2026-03-16T11:37:46Z","failed_at":"e2e","signature_id":"94c04da968f91721cc09a221e720239096d776c4d792c002139186786c45da4a","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:160]: Unexpected error:\n \u003c*errors.joinError | 0xc00150e090\u003e: \n VerifyAllAPIServicesAvailable failed: the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000c42520\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then\",\n err: \u003c*errors.errorString | 0xc000834310\u003e{\n s: \"the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then\",\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:160]: Unexpected error:\n \u003c*errors.joinError | 0xc00150e090\u003e: \n VerifyAllAPIServicesAvailable failed: the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000c42520\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then\",\n err: \u003c*errors.errorString | 0xc000834310\u003e{\n s: \"the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then\",\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"53c8663bb9391640cf5c05f563d13119c0eb49450a0c9d118656d27d9b9f12e7","failure_pattern":"VerifyAllAPIServicesAvailable failed: the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then","search_query":"VerifyAllAPIServicesAvailable failed: the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then"}},{"row_id":"34cdfeefe8e69c5b9a2e19c2b8c0af27e13c93f56294f35b265e4e54b9dca2b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508016086585344","occurred_at":"2026-03-16T11:37:46Z","failed_at":"e2e","signature_id":"ec6f3ab1650305e376ad05b5e2db3a7800bb9480d9593a89b9541996d6f97f23","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0014a61e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-rz587x\" in resourcegroup=\"admin-credential-lifecycle-test-52qk2n\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000e14e80\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-rz587x\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-52qk2n\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e14e60\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-rz587x\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-52qk2n\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e14e40\u003e{\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-rz587x\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-52qk2n\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0014a61e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-rz587x\" in resourcegroup=\"admin-credential-lifecycle-test-52qk2n\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000e14e80\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-rz587x\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-52qk2n\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e14e60\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-rz587x\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-52qk2n\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e14e40\u003e{\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-rz587x\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-52qk2n\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"392f687200777a798775fec34ddd1f45e26ce276ea1385d23f0168734b563cf1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508016086585344","occurred_at":"2026-03-16T11:37:46Z","failed_at":"e2e","signature_id":"6e79391ef4696589d4493e304fb56ecb8c098799e5becd2d150ea03500428d54","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:242]: Unexpected error:\n \u003c*errors.joinError | 0xc000f883a8\u003e: \n context deadline exceeded\n cluster operators not available: image-registry (Available=False, Progressing=True, Degraded=True)\n {\n errs: [\n \u003ccontext.deadlineExceededError\u003e{},\n \u003c*errors.errorString | 0xc000f6f0f0\u003e{\n s: \"cluster operators not available: image-registry (Available=False, Progressing=True, Degraded=True)\",\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:242]: Unexpected error:\n \u003c*errors.joinError | 0xc000f883a8\u003e: \n context deadline exceeded\n cluster operators not available: image-registry (Available=False, Progressing=True, Degraded=True)\n {\n errs: [\n \u003ccontext.deadlineExceededError\u003e{},\n \u003c*errors.errorString | 0xc000f6f0f0\u003e{\n s: \"cluster operators not available: image-registry (Available=False, Progressing=True, Degraded=True)\",\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2a050fd175f9ba873bc55c63e6fe807f7b435ab5054408cba89702d9e47ef41f","failure_pattern":"cluster operators not available: image-registry (Available=False, Progressing=True, Degraded=True)","search_query":"cluster operators not available: image-registry (Available=False, Progressing=True, Degraded=True)"}},{"row_id":"4801783195798be80c30f2e4d4c487e3bb345dfbb4d301c533fa5cf508d6ac27","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508016086585344","occurred_at":"2026-03-16T11:37:46Z","failed_at":"e2e","signature_id":"1b50a5d3e1fe583f1e0265354664b6b2faadbc4b9ee7f6fbdab5a3557ee60a97","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00046a9f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-tfvsrr\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000c039e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-tfvsrr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000c039c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-tfvsrr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000c039a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-tfvsrr\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00046a9f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-tfvsrr\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000c039e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-tfvsrr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000c039c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-tfvsrr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000c039a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-tfvsrr\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"5ceb8d74adab479f081005b33b78baea027297636194e918c3deac0a1e6cf49b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508016086585344","occurred_at":"2026-03-16T11:37:46Z","failed_at":"e2e","signature_id":"186f862347d250481c6ca9d78d60092a19c10a6b576dbe874f88cf19e5faca33","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010da1b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-list9967mr\" in resourcegroup=\"ea-list-qnwxh4\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0008a73c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-list9967mr\\\" in resourcegroup=\\\"ea-list-qnwxh4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008a73a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-list9967mr\\\" in resourcegroup=\\\"ea-list-qnwxh4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008a7380\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-list9967mr\\\" in resourcegroup=\\\"ea-list-qnwxh4\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010da1b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-list9967mr\" in resourcegroup=\"ea-list-qnwxh4\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0008a73c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-list9967mr\\\" in resourcegroup=\\\"ea-list-qnwxh4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008a73a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-list9967mr\\\" in resourcegroup=\\\"ea-list-qnwxh4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008a7380\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-list9967mr\\\" in resourcegroup=\\\"ea-list-qnwxh4\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"5d0571f9bb265ae67b4ca1df1546fda875e2f244f08275fef73d1e4b7f2c2cde","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508016086585344","occurred_at":"2026-03-16T11:37:46Z","failed_at":"e2e","signature_id":"48fd44acc4738fe3677b5c43a45a306cd78b9eb44ec81dfd065f186bd025a736","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ebc4e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-j9km84\" in resourcegroup=\"rg-zstream-upgrade-4-21-6dqm5x\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0011eb0e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-j9km84\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-6dqm5x\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0011eb0c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-j9km84\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-6dqm5x\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0011eb0a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-j9km84\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-6dqm5x\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ebc4e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-j9km84\" in resourcegroup=\"rg-zstream-upgrade-4-21-6dqm5x\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0011eb0e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-j9km84\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-6dqm5x\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0011eb0c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-j9km84\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-6dqm5x\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0011eb0a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-j9km84\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-6dqm5x\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"5fbec9e373ea22a15e029d69a1a4778bf4c1d806d7e9c7139866c389aff779fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508016086585344","occurred_at":"2026-03-16T11:37:46Z","failed_at":"e2e","signature_id":"cc076fffaaeae0df7ae8a919a44a4ff64028f3bdb02e6a268ae866fc085c5dc2","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000dac330\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wtxg8h\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f0ce40\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-wtxg8h\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f0ce20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-wtxg8h\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f0ce00\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-wtxg8h\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000dac330\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wtxg8h\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f0ce40\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-wtxg8h\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f0ce20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-wtxg8h\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f0ce00\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-wtxg8h\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"63671e705788933122a1c7a1842742ca58e06e720e131a0fb6cf017a4da87d45","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508016086585344","occurred_at":"2026-03-16T11:37:46Z","failed_at":"e2e","signature_id":"f5733c0169877eff8c2e84788d5ce6081cdd30a5e7b63706392d2c777d9f91ed","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0011d61b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-qtt68k\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000931660\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-qtt68k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000931640\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-qtt68k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000931620\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-qtt68k\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0011d61b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-qtt68k\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000931660\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-qtt68k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000931640\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-qtt68k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000931620\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-qtt68k\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"6e4083e7392fbbcafbd934d2f4421f2e56b4609ff73a605901c3126179856712","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508016086585344","occurred_at":"2026-03-16T11:37:46Z","failed_at":"e2e","signature_id":"9cba61f6633c23fdb2aa135dc89f0f8636de8fc8e193e5adb85129867a71dca1","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f06138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-zxn7bt\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00086b1c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-zxn7bt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00086b1a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-zxn7bt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00086b180\u003e{\n msg: \"failed waiting for hcpCluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-zxn7bt\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f06138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-zxn7bt\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00086b1c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-zxn7bt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00086b1a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-zxn7bt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00086b180\u003e{\n msg: \"failed waiting for hcpCluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-zxn7bt\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"7128db23df7715980fb446d4f3be3ea9584f44e134174b1c5d65f434d7c785d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508016086585344","occurred_at":"2026-03-16T11:37:46Z","failed_at":"e2e","signature_id":"35e0f69a11ebe2bba5becbc14f12240b80d02fb38077ed676b3184142ef39fc8","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00181c120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-n7mm45\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000fc6dc0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-n7mm45\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000fc6da0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-n7mm45\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000fc6d80\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-n7mm45\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00181c120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-n7mm45\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000fc6dc0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-n7mm45\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000fc6da0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-n7mm45\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000fc6d80\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-n7mm45\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"8ed432ac851d2071162f83678f02de20eecc72aed4d5102ed4783c0a80def053","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508016086585344","occurred_at":"2026-03-16T11:37:46Z","failed_at":"e2e","signature_id":"f8632c4be59ca51d4a3717864a5866624b357a50061edb7da88c0ee993e78de4","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001007530\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-ccmrgc\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a18380\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-ccmrgc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a18280\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-ccmrgc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a18260\u003e{\n msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-ccmrgc\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001007530\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-ccmrgc\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a18380\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-ccmrgc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a18280\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-ccmrgc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a18260\u003e{\n msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-ccmrgc\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"91987cbe8e59717e9f61af6749e1912725d382265e31bb0fe5d32e242e3ec016","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508016086585344","occurred_at":"2026-03-16T11:37:46Z","failed_at":"e2e","signature_id":"d16b4ef51a6b4b673c4fd336229b7c47534983be72a4c27f68c189b6aa8c4735","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:390]: Timed out after 600.001s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:389 with:\nAll ClusterOperators should report Available=True, but these are not available: [image-registry (Available=False)]\nExpected\n \u003c[]string | len:1, cap:1\u003e: [\n \"image-registry (Available=False)\",\n ]\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:390]: Timed out after 600.001s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:389 with:\nAll ClusterOperators should report Available=True, but these are not available: [image-registry (Available=False)]\nExpected\n \u003c[]string | len:1, cap:1\u003e: [\n \"image-registry (Available=False)\",\n ]\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d24e138f3b84b77b0ea6793407ac1f41343d0658f0c69312eac159e4f4f7eb45","failure_pattern":"All ClusterOperators should report Available=True, but these are not available: [image-registry (Available=False)]","search_query":"All ClusterOperators should report Available=True, but these are not available: [image-registry (Available=False)]"}},{"row_id":"a5acd495fb15d9aa4297053ec2c8681d61f46c2d554dd893595509e1f0918a3f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508016086585344","occurred_at":"2026-03-16T11:37:46Z","failed_at":"e2e","signature_id":"7a0814ca8422c9d2afc189f881a9fbd50650d577e535cdb9b1222fa5f6d51d32","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0008f0b28\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-fcpb72\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00096ce60\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fcpb72\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00096ce40\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fcpb72\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00096ce20\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fcpb72\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0008f0b28\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-fcpb72\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00096ce60\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fcpb72\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00096ce40\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fcpb72\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00096ce20\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fcpb72\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"c0485ded67b902a26c3fc3d83a31a3d40173741bd117ae45b81b6f37308a0c90","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508016086585344","occurred_at":"2026-03-16T11:37:46Z","failed_at":"e2e","signature_id":"fe36f7f42c06dc81a753725e67826d4eb587d60c16dce30ce7c7a4541079cfa9","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0016b8108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-19-bmqbjq\" in resourcegroup=\"rg-zstream-upgrade-4-19-6cgn9b\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00011e100\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-19-bmqbjq\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-6cgn9b\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00011e000\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-19-bmqbjq\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-6cgn9b\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007bdf00\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-bmqbjq\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-6cgn9b\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0016b8108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-19-bmqbjq\" in resourcegroup=\"rg-zstream-upgrade-4-19-6cgn9b\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00011e100\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-19-bmqbjq\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-6cgn9b\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00011e000\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-19-bmqbjq\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-6cgn9b\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007bdf00\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-bmqbjq\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-6cgn9b\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"d186922065d480e66cf3c34a806244f493448ae13d8537b11be00443ba03fd00","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508016086585344","occurred_at":"2026-03-16T11:37:46Z","failed_at":"e2e","signature_id":"a7eaf455cd335266442c957dd3de2131127e5cd47b0a3622dabfc6b266803d1d","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00105a1e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-rhfpsw\" in resourcegroup=\"rg-zstream-upgrade-4-20-plc6r2\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000acb700\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-rhfpsw\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-plc6r2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000acb6e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-rhfpsw\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-plc6r2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000acb6c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-20-rhfpsw\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-plc6r2\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00105a1e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-rhfpsw\" in resourcegroup=\"rg-zstream-upgrade-4-20-plc6r2\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000acb700\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-rhfpsw\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-plc6r2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000acb6e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-rhfpsw\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-plc6r2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000acb6c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-20-rhfpsw\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-plc6r2\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"f7a3ccaf7008b6b38194a67470d8f42658b81f261511f5cbb77ecf32d1f3531d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033508016086585344","occurred_at":"2026-03-16T11:37:46Z","failed_at":"e2e","signature_id":"6882e679d4668ebc94332f9c07cf116eb93880332d87c1cb937e76b90c65944a","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010505b8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-swh6nb\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0008a3840\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-swh6nb\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008a3820\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-swh6nb\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008a3800\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-swh6nb\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010505b8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-swh6nb\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0008a3840\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-swh6nb\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008a3820\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-swh6nb\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008a3800\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-swh6nb\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":16,"failure_pattern_ids":["2a050fd175f9ba873bc55c63e6fe807f7b435ab5054408cba89702d9e47ef41f","53c8663bb9391640cf5c05f563d13119c0eb49450a0c9d118656d27d9b9f12e7","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","d24e138f3b84b77b0ea6793407ac1f41343d0658f0c69312eac159e4f4f7eb45"],"matched_occurrences":16,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4461/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507992975970304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4461,"pr_state":"closed","pr_sha":"b806d7769e69345698f74d7a23f602c5bb2a17f2","final_merged_sha":"9baa329552156f993c1fc5b382b78423cd754fef","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:37:39Z"},"failed_at":["e2e"],"failed_test_count":6,"occurrences":[{"row_id":"0b2272e8b4decad524a1033fd946e27faeb88cfacb8b541745461e74d6ca400a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4461/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507992975970304","occurred_at":"2026-03-16T11:37:39Z","failed_at":"e2e","signature_id":"d82cd07c7b6e9370852e605a7cc75c54818eba5bca5be0f7f829a8f2a520c70a","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000782588\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-4m5nhm\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001552de0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-4m5nhm\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001552dc0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-4m5nhm\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001552da0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-4m5nhm\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000782588\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-4m5nhm\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001552de0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-4m5nhm\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001552dc0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-4m5nhm\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001552da0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-4m5nhm\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"1056419621b9f540aa574027eb2dd57668ade9f83051948352a37c3a6d81a434","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4461/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507992975970304","occurred_at":"2026-03-16T11:37:39Z","failed_at":"e2e","signature_id":"a249c5da4a31ece5aace5769734e4a6191e1b2882b655e45f0ef052c3c6cf502","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00145ebe0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-jz875n\" to finish creating: GET https://rp.j5970304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e24d76ce-266a-4022-a92f-2391c52a1890\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e24d76ce-266a-4022-a92f-2391c52a1890\",\n \"name\": \"e24d76ce-266a-4022-a92f-2391c52a1890\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:53:41.22993466Z\",\n \"endTime\": \"2026-03-16T12:54:45.622432365Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-165/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-165/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-jz875n\\\" to finish creating: GET https://rp.j5970304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e24d76ce-266a-4022-a92f-2391c52a1890\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e24d76ce-266a-4022-a92f-2391c52a1890\\\",\\n \\\"name\\\": \\\"e24d76ce-266a-4022-a92f-2391c52a1890\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:53:41.22993466Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:54:45.622432365Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-165/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-165/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00145ebc0\u003e{\n msg: \"failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-jz875n\\\" to finish creating: GET https://rp.j5970304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e24d76ce-266a-4022-a92f-2391c52a1890\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e24d76ce-266a-4022-a92f-2391c52a1890\\\",\\n \\\"name\\\": \\\"e24d76ce-266a-4022-a92f-2391c52a1890\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:53:41.22993466Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:54:45.622432365Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-165/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-165/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00145ebe0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-jz875n\" to finish creating: GET https://rp.j5970304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e24d76ce-266a-4022-a92f-2391c52a1890\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e24d76ce-266a-4022-a92f-2391c52a1890\",\n \"name\": \"e24d76ce-266a-4022-a92f-2391c52a1890\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:53:41.22993466Z\",\n \"endTime\": \"2026-03-16T12:54:45.622432365Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-165/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-165/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-jz875n\\\" to finish creating: GET https://rp.j5970304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e24d76ce-266a-4022-a92f-2391c52a1890\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e24d76ce-266a-4022-a92f-2391c52a1890\\\",\\n \\\"name\\\": \\\"e24d76ce-266a-4022-a92f-2391c52a1890\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:53:41.22993466Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:54:45.622432365Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-165/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-165/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00145ebc0\u003e{\n msg: \"failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-jz875n\\\" to finish creating: GET https://rp.j5970304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e24d76ce-266a-4022-a92f-2391c52a1890\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e24d76ce-266a-4022-a92f-2391c52a1890\\\",\\n \\\"name\\\": \\\"e24d76ce-266a-4022-a92f-2391c52a1890\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:53:41.22993466Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:54:45.622432365Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-165/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-165/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4c879e9967d2d6d016fcade3a8b622ccfbfdacd656377716a48cfaa53eb2c12f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4461/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507992975970304","occurred_at":"2026-03-16T11:37:39Z","failed_at":"e2e","signature_id":"1a7bd44a49bf091e35241ecabb6023006f9e07a0e0766a14dc55f563f94b6e05","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00104cdb0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-dlj4bq\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00075b180\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-dlj4bq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00075b160\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-dlj4bq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00075b140\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-dlj4bq\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00104cdb0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-dlj4bq\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00075b180\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-dlj4bq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00075b160\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-dlj4bq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00075b140\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-dlj4bq\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"6560c87f2680568c1e55801a86f1a67623e983f00495c401f82a752379c98e0f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4461/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507992975970304","occurred_at":"2026-03-16T11:37:39Z","failed_at":"e2e","signature_id":"dd08fdd42478ea5617f8df33c2a3d1222cc892dfbc71e9203cfcaecf10610b7b","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00012a678\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-qr9pqj\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f233c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-qr9pqj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f233a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-qr9pqj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f23380\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-qr9pqj\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00012a678\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-qr9pqj\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f233c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-qr9pqj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f233a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-qr9pqj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f23380\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-qr9pqj\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"cf2ca9d2b325be03c23b29f7200b43b1cd4f91864b0dcbe04efa71417b7b8707","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4461/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507992975970304","occurred_at":"2026-03-16T11:37:39Z","failed_at":"e2e","signature_id":"169046a5edebbcc3f1fb526aef025ab3552e55e48b2abf597f2f78ed24d85d49","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0009d9488\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-6k2ctj\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001037060\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-6k2ctj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001037040\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-6k2ctj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001037020\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-6k2ctj\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0009d9488\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-6k2ctj\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001037060\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-6k2ctj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001037040\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-6k2ctj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001037020\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-6k2ctj\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"f1881c678f82e3916ad97e7fb37884a23e99648c6e7cb9901a1798f4bdf0c237","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4461/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507992975970304","occurred_at":"2026-03-16T11:37:39Z","failed_at":"e2e","signature_id":"7bdc120add7fea0d7334e9acd5f365eea05c89900f006d78eb75c6fbb25e086e","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000c25460\u003e: \n failed to create HCP cluster cluster-zstream-4-20-nr7tz2: failed waiting for cluster=\"cluster-zstream-4-20-nr7tz2\" in resourcegroup=\"rg-zstream-upgrade-4-20-tn8sgk\" to finish creating: GET https://rp.j5970304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/65003440-c6e7-4326-9aee-dee23e620a5b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/65003440-c6e7-4326-9aee-dee23e620a5b\",\n \"name\": \"65003440-c6e7-4326-9aee-dee23e620a5b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:34:40.554390943Z\",\n \"endTime\": \"2026-03-16T12:35:14.145960481Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-237/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-237/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-20-nr7tz2: failed waiting for cluster=\\\"cluster-zstream-4-20-nr7tz2\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tn8sgk\\\" to finish creating: GET https://rp.j5970304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/65003440-c6e7-4326-9aee-dee23e620a5b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/65003440-c6e7-4326-9aee-dee23e620a5b\\\",\\n \\\"name\\\": \\\"65003440-c6e7-4326-9aee-dee23e620a5b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:34:40.554390943Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:35:14.145960481Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-237/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-237/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000c25440\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-nr7tz2\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tn8sgk\\\" to finish creating: GET https://rp.j5970304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/65003440-c6e7-4326-9aee-dee23e620a5b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/65003440-c6e7-4326-9aee-dee23e620a5b\\\",\\n \\\"name\\\": \\\"65003440-c6e7-4326-9aee-dee23e620a5b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:34:40.554390943Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:35:14.145960481Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-237/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-237/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: Subsc...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000c25460\u003e: \n failed to create HCP cluster cluster-zstream-4-20-nr7tz2: failed waiting for cluster=\"cluster-zstream-4-20-nr7tz2\" in resourcegroup=\"rg-zstream-upgrade-4-20-tn8sgk\" to finish creating: GET https://rp.j5970304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/65003440-c6e7-4326-9aee-dee23e620a5b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/65003440-c6e7-4326-9aee-dee23e620a5b\",\n \"name\": \"65003440-c6e7-4326-9aee-dee23e620a5b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:34:40.554390943Z\",\n \"endTime\": \"2026-03-16T12:35:14.145960481Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-237/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-237/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-20-nr7tz2: failed waiting for cluster=\\\"cluster-zstream-4-20-nr7tz2\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tn8sgk\\\" to finish creating: GET https://rp.j5970304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/65003440-c6e7-4326-9aee-dee23e620a5b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/65003440-c6e7-4326-9aee-dee23e620a5b\\\",\\n \\\"name\\\": \\\"65003440-c6e7-4326-9aee-dee23e620a5b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:34:40.554390943Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:35:14.145960481Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-237/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-237/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000c25440\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-nr7tz2\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tn8sgk\\\" to finish creating: GET https://rp.j5970304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/65003440-c6e7-4326-9aee-dee23e620a5b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/65003440-c6e7-4326-9aee-dee23e620a5b\\\",\\n \\\"name\\\": \\\"65003440-c6e7-4326-9aee-dee23e620a5b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:34:40.554390943Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:35:14.145960481Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-237/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-237/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: Subsc...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507991302443008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4463,"pr_state":"open","pr_sha":"8bf53059836d7c658117add712660a10ebbab48b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:37:38Z"},"failed_at":["e2e"],"failed_test_count":25,"occurrences":[{"row_id":"0dce0841e2d7d45eb118265154086f7c3f756a633ec9e984921c753217383f3f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507991302443008","occurred_at":"2026-03-16T11:37:38Z","failed_at":"e2e","signature_id":"339a31299a71e81715aa7ca286ee92bdcc842e90c88a692eb143da7112491382","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:124]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e46440\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-pullsecret-hcp-cluster-27cvtn\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-pullsecret-hcp-cluster-27cvtn/operationStatuses/08584279348209008846\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-sxgdnx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-p2h32vyw2l2ma\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-pullsecret-hcp-cluster-27cvtn\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-pullsecret-hcp-cluster-27cvtn/operationStatuses/08584279348209008846\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-sxgdnx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-p2h32vyw2l2ma\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e46420\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-pullsecret-hcp-cluster-27cvtn\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-pullsecret-hcp-cluster-27cvtn/operationS...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:124]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e46440\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-pullsecret-hcp-cluster-27cvtn\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-pullsecret-hcp-cluster-27cvtn/operationStatuses/08584279348209008846\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-sxgdnx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-p2h32vyw2l2ma\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-pullsecret-hcp-cluster-27cvtn\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-pullsecret-hcp-cluster-27cvtn/operationStatuses/08584279348209008846\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-sxgdnx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-p2h32vyw2l2ma\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e46420\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-pullsecret-hcp-cluster-27cvtn\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-pullsecret-hcp-cluster-27cvtn/operationS...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"103e8bb54047fdf6cfe0224067aff4e5252fcbf65d1137b53defa0d64a8b3f7d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507991302443008","occurred_at":"2026-03-16T11:37:38Z","failed_at":"e2e","signature_id":"19377fc9ccb5d89eecde1cbd348440b4b4e751a9bc12fcfe5d5095949adfd402","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:77]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a61080\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-list-test-cluster-m2fjfs-m7twnq\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-list-test-cluster-m2fjfs-m7twnq/operationStatuses/08584279348212172014\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-h64t8x/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3o26pq5ewqal4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-list-test-cluster-m2fjfs-m7twnq\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-list-test-cluster-m2fjfs-m7twnq/operationStatuses/08584279348212172014\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-h64t8x/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3o26pq5ewqal4\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a61020\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-list-test-cluster-m2fjfs-m7twnq\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-list-test-cluster-m2fjfs-m7twnq/op...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:77]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a61080\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-list-test-cluster-m2fjfs-m7twnq\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-list-test-cluster-m2fjfs-m7twnq/operationStatuses/08584279348212172014\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-h64t8x/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3o26pq5ewqal4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-list-test-cluster-m2fjfs-m7twnq\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-list-test-cluster-m2fjfs-m7twnq/operationStatuses/08584279348212172014\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-h64t8x/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3o26pq5ewqal4\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a61020\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-list-test-cluster-m2fjfs-m7twnq\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-list-test-cluster-m2fjfs-m7twnq/op...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"13735d96251674e6659e51ff94ef4326b6efe36367459b76a0e7041eb58923fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507991302443008","occurred_at":"2026-03-16T11:37:38Z","failed_at":"e2e","signature_id":"d18a44564cd1ae8156ec3055605880d33eec3f1a04ef285db136107c24bebb49","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:71]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b84a00\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-hcp-cluster-np-128-x9qxl8\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-hcp-cluster-np-128-x9qxl8/operationStatuses/08584279348102075889\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-dv8nvm/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3ljyjjx2n7gxo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-hcp-cluster-np-128-x9qxl8\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-hcp-cluster-np-128-x9qxl8/operationStatuses/08584279348102075889\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-dv8nvm/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3ljyjjx2n7gxo\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can b...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:71]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b84a00\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-hcp-cluster-np-128-x9qxl8\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-hcp-cluster-np-128-x9qxl8/operationStatuses/08584279348102075889\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-dv8nvm/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3ljyjjx2n7gxo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-hcp-cluster-np-128-x9qxl8\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-hcp-cluster-np-128-x9qxl8/operationStatuses/08584279348102075889\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-dv8nvm/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3ljyjjx2n7gxo\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can b...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"2d07e84d23c94090f920f6404ced6332ba61a639a5d2b031865c4f7107fe9e74","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507991302443008","occurred_at":"2026-03-16T11:37:38Z","failed_at":"e2e","signature_id":"1005d12f90e4230f92c0a1db0895d1457cad668b45a17adc5f5c155253b1d871","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:82]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b00720\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-admin-cred-lifecycle-bgf52c-s8crhw\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-admin-cred-lifecycle-bgf52c-s8crhw/operationStatuses/08584279348109248089\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-sffqpz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-cxue26wkwsw7m\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-146/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-admin-cred-lifecycle-bgf52c-s8crhw\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-admin-cred-lifecycle-bgf52c-s8crhw/operationStatuses/08584279348109248089\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-sffqpz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-cxue26wkwsw7m\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:82]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b00720\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-admin-cred-lifecycle-bgf52c-s8crhw\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-admin-cred-lifecycle-bgf52c-s8crhw/operationStatuses/08584279348109248089\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-sffqpz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-cxue26wkwsw7m\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-146/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-admin-cred-lifecycle-bgf52c-s8crhw\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-admin-cred-lifecycle-bgf52c-s8crhw/operationStatuses/08584279348109248089\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-sffqpz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-cxue26wkwsw7m\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"2df937d22454a98de82add4865c828f1473e14c67dc39bd31c4fabb2f8228e31","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507991302443008","occurred_at":"2026-03-16T11:37:38Z","failed_at":"e2e","signature_id":"11e4305d6af5d542b6641838030dc0af52da4f388a8048c1e15cf013cf211330","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:96]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00157d1e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-cluster-pkz4xl\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-cluster-pkz4xl/operationStatuses/08584279348212078178\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-lqwnfc/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j3gkygvtvkvjs\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-basic-cluster-pkz4xl\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-cluster-pkz4xl/operationStatuses/08584279348212078178\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-lqwnfc/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j3gkygvtvkvjs\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00157d1c0\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-basic-cluster-pkz4xl\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-cluster-pkz4xl/operationStatuses/0858427934821207...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:96]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00157d1e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-cluster-pkz4xl\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-cluster-pkz4xl/operationStatuses/08584279348212078178\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-lqwnfc/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j3gkygvtvkvjs\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-basic-cluster-pkz4xl\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-cluster-pkz4xl/operationStatuses/08584279348212078178\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-lqwnfc/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j3gkygvtvkvjs\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00157d1c0\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-basic-cluster-pkz4xl\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-cluster-pkz4xl/operationStatuses/0858427934821207...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"33a325078e1159ca60e8fab9d482d396c5cfb383f435580e7ec856eaf77bc425","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507991302443008","occurred_at":"2026-03-16T11:37:38Z","failed_at":"e2e","signature_id":"6f5d19790d1913ee72dc28e7b141c2414bcac8e70cb31e7252f719828e496db3","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:117]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a48d40\u003e: \n failed waiting for deployment \"mi-cluster-ver-4-19-kq4rvt\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-ver-4-19-kq4rvt/operationStatuses/08584279348212264473\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-mj74kg/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-258/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed waiting for deployment \\\"mi-cluster-ver-4-19-kq4rvt\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-ver-4-19-kq4rvt/operationStatuses/08584279348212264473\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-mj74kg/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimi...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:117]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a48d40\u003e: \n failed waiting for deployment \"mi-cluster-ver-4-19-kq4rvt\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-ver-4-19-kq4rvt/operationStatuses/08584279348212264473\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-mj74kg/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-258/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed waiting for deployment \\\"mi-cluster-ver-4-19-kq4rvt\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-ver-4-19-kq4rvt/operationStatuses/08584279348212264473\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-mj74kg/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimi...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"36b9ae4ef7539f528162f43506bc6de61b635585221933c15b47f2ab8945a9c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507991302443008","occurred_at":"2026-03-16T11:37:38Z","failed_at":"e2e","signature_id":"ac25e0a34da073c9787c46423356529f2492ee775b885f5381ef25e1430f5b3c","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:88]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000876f00\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-autoscaling-hcp-cluster-4bjpnq\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-autoscaling-hcp-cluster-4bjpnq/operationStatuses/08584279348105316583\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-174/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7kqnsz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-e4ixwyxsfjnje\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-autoscaling-hcp-cluster-4bjpnq\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-autoscaling-hcp-cluster-4bjpnq/operationStatuses/08584279348105316583\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-174/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7kqnsz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-e4ixwyxsfjnje\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:88]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000876f00\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-autoscaling-hcp-cluster-4bjpnq\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-autoscaling-hcp-cluster-4bjpnq/operationStatuses/08584279348105316583\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-174/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7kqnsz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-e4ixwyxsfjnje\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-autoscaling-hcp-cluster-4bjpnq\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-autoscaling-hcp-cluster-4bjpnq/operationStatuses/08584279348105316583\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-174/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7kqnsz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-e4ixwyxsfjnje\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"4f0c6727824519d2573d7f620fbce1e8da55c2804b30cdb7463ffbfaebb5443c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507991302443008","occurred_at":"2026-03-16T11:37:38Z","failed_at":"e2e","signature_id":"05f963a023e5b60c829c6615221ff22eb4cb835a570e67232545bbc695925174","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:80]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e0ebc0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-arm64-vm-hcp-cluster-6bz7gn\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-arm64-vm-hcp-cluster-6bz7gn/operationStatuses/08584279348007219168\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-tfhhvz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6htyoibevj7ye\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-arm64-vm-hcp-cluster-6bz7gn\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-arm64-vm-hcp-cluster-6bz7gn/operationStatuses/08584279348007219168\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-tfhhvz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6htyoibevj7ye\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e0eba0\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-arm64-vm-hcp-cluster-6bz7gn\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-arm64-vm-hcp-cluster-6bz7gn/operationStatuses...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:80]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e0ebc0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-arm64-vm-hcp-cluster-6bz7gn\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-arm64-vm-hcp-cluster-6bz7gn/operationStatuses/08584279348007219168\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-tfhhvz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6htyoibevj7ye\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-arm64-vm-hcp-cluster-6bz7gn\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-arm64-vm-hcp-cluster-6bz7gn/operationStatuses/08584279348007219168\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-tfhhvz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6htyoibevj7ye\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e0eba0\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-arm64-vm-hcp-cluster-6bz7gn\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-arm64-vm-hcp-cluster-6bz7gn/operationStatuses...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"54f76f3462177d564a2271323bb44807211cd2fbf6bd1bfb625b33e41f887e41","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507991302443008","occurred_at":"2026-03-16T11:37:38Z","failed_at":"e2e","signature_id":"2f1518253caaafd823c45a518726bccb699446b6d1401523fd799d0499bd59af","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:77]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b081c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-oidc-wi-cluster-8dntxt\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-oidc-wi-cluster-8dntxt/operationStatuses/08584279346243357195\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-swjt27/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-s6kwp7xg43dbi\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-11/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-oidc-wi-cluster-8dntxt\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-oidc-wi-cluster-8dntxt/operationStatuses/08584279346243357195\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-swjt27/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-s6kwp7xg43dbi\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/974ebd4...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:77]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b081c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-oidc-wi-cluster-8dntxt\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-oidc-wi-cluster-8dntxt/operationStatuses/08584279346243357195\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-swjt27/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-s6kwp7xg43dbi\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-11/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-oidc-wi-cluster-8dntxt\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-oidc-wi-cluster-8dntxt/operationStatuses/08584279346243357195\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-swjt27/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-s6kwp7xg43dbi\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/974ebd4...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"5a6c0b568b62328ebf2e45e1060a29adb2940e8f65e41d53aca179fb60061fef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507991302443008","occurred_at":"2026-03-16T11:37:38Z","failed_at":"e2e","signature_id":"fdaa17ef3c8ed795953e02a901cd9f52e6422d06a70c08cf9917fe4ab6fc222b","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:79]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b14740\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-negative-tests-cluster-8kwprr\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-negative-tests-cluster-8kwprr/operationStatuses/08584279346495377975\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-268/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-br4jhg/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oeayaalhj7wjy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-negative-tests-cluster-8kwprr\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-negative-tests-cluster-8kwprr/operationStatuses/08584279346495377975\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-268/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:79]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b14740\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-negative-tests-cluster-8kwprr\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-negative-tests-cluster-8kwprr/operationStatuses/08584279346495377975\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-268/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-br4jhg/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oeayaalhj7wjy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-negative-tests-cluster-8kwprr\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-negative-tests-cluster-8kwprr/operationStatuses/08584279346495377975\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-268/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"5d2bcb50ca58533dc877373bed36a688256572dbfe9b7bbf7e231d6e062867aa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507991302443008","occurred_at":"2026-03-16T11:37:38Z","failed_at":"e2e","signature_id":"434628566f5c6e4fc1fbe24372036eda427a55e6a10468e662f4914181c9a4a6","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:75]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a804e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ea-listwq6sgv-xvmrt6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-listwq6sgv-xvmrt6/operationStatuses/08584279348213194323\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-klfwp7/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3hysavasweej6\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-ea-listwq6sgv-xvmrt6\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-listwq6sgv-xvmrt6/operationStatuses/08584279348213194323\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-klfwp7/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3hysavasweej6\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a804c0\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-ea-listwq6sgv-xvmrt6\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-listwq6sgv-xvmrt6/operationStatuses/08584279348213194323\\n--------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:75]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a804e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ea-listwq6sgv-xvmrt6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-listwq6sgv-xvmrt6/operationStatuses/08584279348213194323\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-klfwp7/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3hysavasweej6\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-ea-listwq6sgv-xvmrt6\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-listwq6sgv-xvmrt6/operationStatuses/08584279348213194323\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-klfwp7/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3hysavasweej6\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a804c0\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-ea-listwq6sgv-xvmrt6\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-listwq6sgv-xvmrt6/operationStatuses/08584279348213194323\\n--------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"839b05d648db1f49043457fccdc9bfe7ff84ac51d5242c1c1898b86979859e05","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507991302443008","occurred_at":"2026-03-16T11:37:38Z","failed_at":"e2e","signature_id":"a4d6dca4936c87cbbfed3bc9bcfece1700952a87185d998ffc2c6524504f6232","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:77]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0006afbc0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-engineering-hcp-cluster-fwgghx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-engineering-hcp-cluster-fwgghx/operationStatuses/08584279346496343754\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-267/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-engineering-hcp-cluster-fwgghx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-engineering-hcp-cluster-fwgghx/operationStatuses/08584279346496343754\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-267/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0006afba0\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-engineering-hcp-cluster-fwgghx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-engineering-hcp-cluster-fwgghx/operationStatuses/08584279346496343754\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-267/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operati...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:77]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0006afbc0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-engineering-hcp-cluster-fwgghx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-engineering-hcp-cluster-fwgghx/operationStatuses/08584279346496343754\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-267/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-engineering-hcp-cluster-fwgghx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-engineering-hcp-cluster-fwgghx/operationStatuses/08584279346496343754\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-267/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0006afba0\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-engineering-hcp-cluster-fwgghx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-engineering-hcp-cluster-fwgghx/operationStatuses/08584279346496343754\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-267/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operati...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"8978bb8f424fc18ffdf96803d82e6793c246f9f7f10a79868ffcb2b97cd2fc19","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507991302443008","occurred_at":"2026-03-16T11:37:38Z","failed_at":"e2e","signature_id":"2082b75060376d3cb654d05e32086ba44d5d40ac648110c407815b9a21c7d3ff","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:89]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a123a0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cidr-connectivity-test-ztbrzp\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-ztbrzp/operationStatuses/08584279348218393707\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-9lk62w/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wihoigd7cpguy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-11/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cidr-connectivity-test-ztbrzp\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-ztbrzp/operationStatuses/08584279348218393707\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-9lk62w/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wihoigd7cpguy\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\"...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:89]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a123a0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cidr-connectivity-test-ztbrzp\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-ztbrzp/operationStatuses/08584279348218393707\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-9lk62w/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wihoigd7cpguy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-11/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cidr-connectivity-test-ztbrzp\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-ztbrzp/operationStatuses/08584279348218393707\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-9lk62w/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wihoigd7cpguy\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\"...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"8c7fc3d320de280f4d7e325732f67dccb06c7c5859724dcbde02912678b54f8a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507991302443008","occurred_at":"2026-03-16T11:37:38Z","failed_at":"e2e","signature_id":"6c9af334cef7fa22b04e50058955753f88d1bdeb4223040f446af67eb51a075b","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:81]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b11b20\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-sre-hcp-cluster-gblr5n\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-sre-hcp-cluster-gblr5n/operationStatuses/08584279348109332006\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-ngprr7/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-456d7mdgworfy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-sre-hcp-cluster-gblr5n\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-sre-hcp-cluster-gblr5n/operationStatuses/08584279348109332006\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-ngprr7/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-456d7mdgworfy\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000b11b00\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-sre-hcp-cluster-gblr5n\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-sre-hcp-cluster-gblr5n/operationStatuses/085842793481093...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:81]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b11b20\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-sre-hcp-cluster-gblr5n\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-sre-hcp-cluster-gblr5n/operationStatuses/08584279348109332006\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-ngprr7/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-456d7mdgworfy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-sre-hcp-cluster-gblr5n\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-sre-hcp-cluster-gblr5n/operationStatuses/08584279348109332006\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-ngprr7/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-456d7mdgworfy\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000b11b00\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-sre-hcp-cluster-gblr5n\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-sre-hcp-cluster-gblr5n/operationStatuses/085842793481093...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"98153b9af78d7cd0ad7adb4d6af2d1752dac1fadb296b5c2a9f82190a051faa0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507991302443008","occurred_at":"2026-03-16T11:37:38Z","failed_at":"e2e","signature_id":"ccadcc572465faaea64031a3e8284f8ada52580085fb6f86586a22477dd61ad6","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:81]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000f907c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster-sxx96d\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-sxx96d/operationStatuses/08584279348104971436\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-j95xvf/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-eqbgltuyzhpg4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-basic-hcp-cluster-sxx96d\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-sxx96d/operationStatuses/08584279348104971436\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-j95xvf/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-eqbgltuyzhpg4\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000f907a0\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-basic-hcp-cluster-sxx96d\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-sxx96d/operationStatuses/0858427934810497...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:81]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000f907c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster-sxx96d\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-sxx96d/operationStatuses/08584279348104971436\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-j95xvf/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-eqbgltuyzhpg4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-basic-hcp-cluster-sxx96d\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-sxx96d/operationStatuses/08584279348104971436\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-j95xvf/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-eqbgltuyzhpg4\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000f907a0\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-basic-hcp-cluster-sxx96d\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-sxx96d/operationStatuses/0858427934810497...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"a8e49499ca2faddfba6185c33742ab366fa90aa049f906252f10e65eb2310560","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507991302443008","occurred_at":"2026-03-16T11:37:38Z","failed_at":"e2e","signature_id":"7cf5c4941d9cb23dee4a8464077d7367a45c1c2f483e47292970bfe5e68b7445","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000af34c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-19-dqbcpr-xlfb8n\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-19-dqbcpr-xlfb8n/operationStatuses/08584279346240555694\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-16/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-nwdnkm/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-opvghki66zxlk\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-19-dqbcpr-xlfb8n\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-19-dqbcpr-xlfb8n/operationStatuses/08584279346240555694\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-16/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-nwdnkm/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-opvghki66zxlk\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"N...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000af34c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-19-dqbcpr-xlfb8n\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-19-dqbcpr-xlfb8n/operationStatuses/08584279346240555694\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-16/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-nwdnkm/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-opvghki66zxlk\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-19-dqbcpr-xlfb8n\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-19-dqbcpr-xlfb8n/operationStatuses/08584279346240555694\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-16/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-nwdnkm/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-opvghki66zxlk\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"N...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"b0c64f168817573bcaa8fe9b107389370cdb97005794d0c0883b29f7cdf80ef1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507991302443008","occurred_at":"2026-03-16T11:37:38Z","failed_at":"e2e","signature_id":"61858f4c92c6e069b32499b0e47c4d6f0a4204633d98cf53664c3b2c5457b023","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00083b480\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-20-256m6w-k2cwhd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-256m6w-k2cwhd/operationStatuses/08584279346244608183\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-6s49sd/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-k4eojl4c63cvc\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-146/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-20-256m6w-k2cwhd\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-256m6w-k2cwhd/operationStatuses/08584279346244608183\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-6s49sd/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-k4eojl4c63cvc\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00083b480\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-20-256m6w-k2cwhd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-256m6w-k2cwhd/operationStatuses/08584279346244608183\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-6s49sd/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-k4eojl4c63cvc\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-146/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-20-256m6w-k2cwhd\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-256m6w-k2cwhd/operationStatuses/08584279346244608183\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-6s49sd/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-k4eojl4c63cvc\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"b5c32db8bc1a2377dbfca5d9ff4be7f42d2901ecd2f598920f16abb2b782d650","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507991302443008","occurred_at":"2026-03-16T11:37:38Z","failed_at":"e2e","signature_id":"5c5c1f4d79dfd9bfae4697ee29b7a02641c1a207262cb1193e524977d58a7f2a","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:74]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000820840\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-autoscale-cluster-w8zn5h\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-autoscale-cluster-w8zn5h/operationStatuses/08584279346245680689\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-hk2vsv/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ssrwqrnhyyabs\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-174/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-np-autoscale-cluster-w8zn5h\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-autoscale-cluster-w8zn5h/operationStatuses/08584279346245680689\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-hk2vsv/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ssrwqrnhyyabs\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-174/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0008207e0\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-np-autoscale-cluster-w8zn5h\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-autoscale-cluster-w8zn5h/operationStatuses/08584279346245680689\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:74]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000820840\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-autoscale-cluster-w8zn5h\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-autoscale-cluster-w8zn5h/operationStatuses/08584279346245680689\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-hk2vsv/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ssrwqrnhyyabs\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-174/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-np-autoscale-cluster-w8zn5h\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-autoscale-cluster-w8zn5h/operationStatuses/08584279346245680689\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-hk2vsv/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ssrwqrnhyyabs\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-174/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0008207e0\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-np-autoscale-cluster-w8zn5h\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-autoscale-cluster-w8zn5h/operationStatuses/08584279346245680689\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"b6fa981f6560040b61377565b45c5443f12b5f956e83afd303e91568fe46da62","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507991302443008","occurred_at":"2026-03-16T11:37:38Z","failed_at":"e2e","signature_id":"95f84be2d281c6a9ff8be9bf1ee36483a3c0d66e0fb01d2f92e0955ac8bef203","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:79]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00079f700\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-disabled-image-registry-hcp-cluster-9zlgfx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-disabled-image-registry-hcp-cluster-9zlgfx/operationStatuses/08584279348102707114\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-16/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-2bdlsk/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-qmh74zogaodhg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-disabled-image-registry-hcp-cluster-9zlgfx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-disabled-image-registry-hcp-cluster-9zlgfx/operationStatuses/08584279348102707114\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-16/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-2bdlsk/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-qmh74zogaodhg\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"me...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:79]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00079f700\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-disabled-image-registry-hcp-cluster-9zlgfx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-disabled-image-registry-hcp-cluster-9zlgfx/operationStatuses/08584279348102707114\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-16/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-2bdlsk/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-qmh74zogaodhg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-disabled-image-registry-hcp-cluster-9zlgfx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-disabled-image-registry-hcp-cluster-9zlgfx/operationStatuses/08584279348102707114\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-16/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-2bdlsk/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-qmh74zogaodhg\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"me...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"b9317994ec3197f4e0dcabc7257bfc9d3252dfeb00d716ea9bc19afa185e79ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507991302443008","occurred_at":"2026-03-16T11:37:38Z","failed_at":"e2e","signature_id":"fa3c2c1a6090e64676cd8282a4d71d1eecb738c4ceb987efcd4b47f171f52cf4","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:86]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0015bada0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster-r466zr\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-r466zr/operationStatuses/08584279348107332186\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-q5zbx8/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-eqbgltuyzhpg4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-224/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-basic-hcp-cluster-r466zr\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-r466zr/operationStatuses/08584279348107332186\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-q5zbx8/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-eqbgltuyzhpg4\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:86]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0015bada0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster-r466zr\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-r466zr/operationStatuses/08584279348107332186\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-q5zbx8/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-eqbgltuyzhpg4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-224/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-basic-hcp-cluster-r466zr\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-r466zr/operationStatuses/08584279348107332186\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-q5zbx8/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-eqbgltuyzhpg4\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"d1372dbedfd60461ca3771971ab88b7147d32e4c7e4e6424a06d9508a72c2c10","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507991302443008","occurred_at":"2026-03-16T11:37:38Z","failed_at":"e2e","signature_id":"fce7f9bcb434429de15878fcc5067564516d28b1640a697e017f65c7cc26ce27","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000ae6600\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-21-twvswg-cnmbcp\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-twvswg-cnmbcp/operationStatuses/08584279346249234412\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-2/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-8dx8gm/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-acmzzmpfurenm\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-21-twvswg-cnmbcp\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-twvswg-cnmbcp/operationStatuses/08584279346249234412\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-2/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-8dx8gm/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-acmzzmpfurenm\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000ae6600\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-21-twvswg-cnmbcp\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-twvswg-cnmbcp/operationStatuses/08584279346249234412\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-2/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-8dx8gm/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-acmzzmpfurenm\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-21-twvswg-cnmbcp\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-twvswg-cnmbcp/operationStatuses/08584279346249234412\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-2/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-8dx8gm/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-acmzzmpfurenm\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"ebc0ee668a2a2a808706c5b98023882cc07e991e45c8ffa48e81ee23ad7596ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507991302443008","occurred_at":"2026-03-16T11:37:38Z","failed_at":"e2e","signature_id":"72e0ae2c447058d8152c8c4da9bc12d55f5e010146a8b759859113c457e17840","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:158]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a82980\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-node-limit-cluster-9cq4wv\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-9cq4wv/operationStatuses/08584279346246022131\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8bkmwp/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-277/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-node-limit-cluster-9cq4wv\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-9cq4wv/operationStatuses/08584279346246022131\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8bkmwp/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscri...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:158]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a82980\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-node-limit-cluster-9cq4wv\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-9cq4wv/operationStatuses/08584279346246022131\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8bkmwp/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-277/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-node-limit-cluster-9cq4wv\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-9cq4wv/operationStatuses/08584279346246022131\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8bkmwp/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscri...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"f12acccba8a39a3c0d152e9182f04f36613df447f95759f254cee101ce8bb3af","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507991302443008","occurred_at":"2026-03-16T11:37:38Z","failed_at":"e2e","signature_id":"ea9947ec36d5e956a31a392c8b4ec525418f3ce2dbe19251afcc782077fd86ff","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:150]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001602800\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-patch-tags-cluster-9lg58m\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-patch-tags-cluster-9lg58m/operationStatuses/08584279348211361650\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-lvsw6z/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fklo75htpxzxc\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-67/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-patch-tags-cluster-9lg58m\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-patch-tags-cluster-9lg58m/operationStatuses/08584279348211361650\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-lvsw6z/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fklo75htpxzxc\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscription...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:150]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001602800\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-patch-tags-cluster-9lg58m\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-patch-tags-cluster-9lg58m/operationStatuses/08584279348211361650\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-lvsw6z/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fklo75htpxzxc\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-67/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-patch-tags-cluster-9lg58m\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-patch-tags-cluster-9lg58m/operationStatuses/08584279348211361650\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-lvsw6z/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fklo75htpxzxc\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscription...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"f771e297246eed06e6745fdbde4afe6e6ebc3eebffe173a84551b17a83af4caa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507991302443008","occurred_at":"2026-03-16T11:37:38Z","failed_at":"e2e","signature_id":"1778339bdff6fa669d1b89c9a69dcbfb826b9154d101f23e36863b251debe237","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:73]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00069ef40\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-update-nodes-hcp-cluster-h6rx76\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-h6rx76/operationStatuses/08584279346137168669\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-258/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-nvczjs/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-np-update-nodes-hcp-cluster-h6rx76\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-h6rx76/operationStatuses/08584279346137168669\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-258/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-nvczjs/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:73]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00069ef40\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-update-nodes-hcp-cluster-h6rx76\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-h6rx76/operationStatuses/08584279346137168669\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-258/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-nvczjs/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-np-update-nodes-hcp-cluster-h6rx76\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-update-nodes-hcp-cluster-h6rx76/operationStatuses/08584279346137168669\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-258/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-nvczjs/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w3k44uxsb3qey\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"ff328de598ae3e3b37aa14c3f8a529301b64f32d8cd85bc07be5f7fd71dc5051","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507991302443008","occurred_at":"2026-03-16T11:37:38Z","failed_at":"e2e","signature_id":"929c205ec9de86a2420442143c67c4f914a61c1e4d95a2227ac9a82e0c2676a6","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:84]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b83260\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ea-cluster-6tljp8\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-cluster-6tljp8/operationStatuses/08584279348216220124\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-6rg4hn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ltdphft7fgrwg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-ea-cluster-6tljp8\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-cluster-6tljp8/operationStatuses/08584279348216220124\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-6rg4hn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ltdphft7fgrwg\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000b83240\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-ea-cluster-6tljp8\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-cluster-6tljp8/operationStatuses/08584279348216220124\\n------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:84]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b83260\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ea-cluster-6tljp8\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-cluster-6tljp8/operationStatuses/08584279348216220124\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-6rg4hn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ltdphft7fgrwg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-ea-cluster-6tljp8\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-cluster-6tljp8/operationStatuses/08584279348216220124\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-6rg4hn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ltdphft7fgrwg\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000b83240\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-ea-cluster-6tljp8\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-cluster-6tljp8/operationStatuses/08584279348216220124\\n------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":25,"failure_pattern_ids":["82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d"],"matched_occurrences":25,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4465/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507978727919616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4465,"pr_state":"closed","pr_sha":"842e66a08ab754c171642e7075dee98285c9e52b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:37:31Z"},"failed_at":["e2e"],"failed_test_count":14,"occurrences":[{"row_id":"0b4da3059e7a550645d06c9df9f126f3e99b704dd92492b489ee81cb948a462c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4465/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507978727919616","occurred_at":"2026-03-16T11:37:31Z","failed_at":"e2e","signature_id":"3b2c249a340e174ba7323a2f20de3d76ad85bca0518b6bfadf905bdc74cb435b","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e2a210\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-m7l49f\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a1a8e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-m7l49f\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a1a840\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-m7l49f\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a1a780\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-m7l49f\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e2a210\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-m7l49f\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a1a8e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-m7l49f\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a1a840\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-m7l49f\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a1a780\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-m7l49f\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"343800bd0d37b96c43b541bd374cfd2bed5819c25282b5a42746584f3e20abdd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4465/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507978727919616","occurred_at":"2026-03-16T11:37:31Z","failed_at":"e2e","signature_id":"e0935888b290aa300b6a96ff4100539b768907aa896d2d606bd43461608e8adb","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000440320\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-bc8srv\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cdc41c88-18b6-498f-8f84-982adcd4f3fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cdc41c88-18b6-498f-8f84-982adcd4f3fd\",\n \"name\": \"cdc41c88-18b6-498f-8f84-982adcd4f3fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:41:45.16428613Z\",\n \"endTime\": \"2026-03-16T12:42:47.486090571Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-bc8srv\\\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cdc41c88-18b6-498f-8f84-982adcd4f3fd\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cdc41c88-18b6-498f-8f84-982adcd4f3fd\\\",\\n \\\"name\\\": \\\"cdc41c88-18b6-498f-8f84-982adcd4f3fd\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:41:45.16428613Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:42:47.486090571Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0004402c0\u003e{\n msg: \"failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-bc8srv\\\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cdc41c88-18b6-498f-8f84-982adcd4f3fd\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cdc41c88-18b6-498f-8f84-982adcd4f3fd\\\",\\n \\\"name\\\": \\\"cdc41c88-18b6-498f-8f84-982adcd4f3fd\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:41:45.16428613Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:42:47.486090571Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n---------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000440320\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-bc8srv\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cdc41c88-18b6-498f-8f84-982adcd4f3fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cdc41c88-18b6-498f-8f84-982adcd4f3fd\",\n \"name\": \"cdc41c88-18b6-498f-8f84-982adcd4f3fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:41:45.16428613Z\",\n \"endTime\": \"2026-03-16T12:42:47.486090571Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-bc8srv\\\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cdc41c88-18b6-498f-8f84-982adcd4f3fd\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cdc41c88-18b6-498f-8f84-982adcd4f3fd\\\",\\n \\\"name\\\": \\\"cdc41c88-18b6-498f-8f84-982adcd4f3fd\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:41:45.16428613Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:42:47.486090571Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0004402c0\u003e{\n msg: \"failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-bc8srv\\\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cdc41c88-18b6-498f-8f84-982adcd4f3fd\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cdc41c88-18b6-498f-8f84-982adcd4f3fd\\\",\\n \\\"name\\\": \\\"cdc41c88-18b6-498f-8f84-982adcd4f3fd\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:41:45.16428613Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:42:47.486090571Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n---------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"354f19585f7df7f220afb6ab3e0ec7f12407374a47fad2ef58cac452954f1d23","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4465/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507978727919616","occurred_at":"2026-03-16T11:37:31Z","failed_at":"e2e","signature_id":"bda2b2c13d721b873edeee0701ac5aa478ea0a6bdae438a2b214c5613603990e","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0007bc4c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-qmb28v\" in resourcegroup=\"rg-zstream-upgrade-4-20-2v76gp\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0006f7ca0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-qmb28v\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-2v76gp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006f7c80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-qmb28v\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-2v76gp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006f7900\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-20-qmb28v\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-2v76gp\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0007bc4c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-qmb28v\" in resourcegroup=\"rg-zstream-upgrade-4-20-2v76gp\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0006f7ca0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-qmb28v\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-2v76gp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006f7c80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-qmb28v\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-2v76gp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006f7900\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-20-qmb28v\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-2v76gp\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"4886c4cd69c82289521797198928605db10b439d8a2ad794141944f2eabb8e98","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4465/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507978727919616","occurred_at":"2026-03-16T11:37:31Z","failed_at":"e2e","signature_id":"2bb5533ba101946c91015f876a00dbc8fcfd7def68fcc463482524d556c08001","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:86]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0008b7280\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-7rjp56\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/164046a3-fff4-4265-b9e3-481353a864f9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/164046a3-fff4-4265-b9e3-481353a864f9\",\n \"name\": \"164046a3-fff4-4265-b9e3-481353a864f9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:46:46.289122417Z\",\n \"endTime\": \"2026-03-16T12:47:33.920097986Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-7rjp56\\\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/164046a3-fff4-4265-b9e3-481353a864f9\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/164046a3-fff4-4265-b9e3-481353a864f9\\\",\\n \\\"name\\\": \\\"164046a3-fff4-4265-b9e3-481353a864f9\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:46:46.289122417Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:47:33.920097986Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0008b7260\u003e{\n msg: \"failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-7rjp56\\\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/164046a3-fff4-4265-b9e3-481353a864f9\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/164046a3-fff4-4265-b9e3-481353a864f9\\\",\\n \\\"name\\\": \\\"164046a3-fff4-4265-b9e3-481353a864f9\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:46:46.289122417Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:47:33.920097986Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-------------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:86]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0008b7280\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-7rjp56\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/164046a3-fff4-4265-b9e3-481353a864f9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/164046a3-fff4-4265-b9e3-481353a864f9\",\n \"name\": \"164046a3-fff4-4265-b9e3-481353a864f9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:46:46.289122417Z\",\n \"endTime\": \"2026-03-16T12:47:33.920097986Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-7rjp56\\\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/164046a3-fff4-4265-b9e3-481353a864f9\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/164046a3-fff4-4265-b9e3-481353a864f9\\\",\\n \\\"name\\\": \\\"164046a3-fff4-4265-b9e3-481353a864f9\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:46:46.289122417Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:47:33.920097986Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0008b7260\u003e{\n msg: \"failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-7rjp56\\\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/164046a3-fff4-4265-b9e3-481353a864f9\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/164046a3-fff4-4265-b9e3-481353a864f9\\\",\\n \\\"name\\\": \\\"164046a3-fff4-4265-b9e3-481353a864f9\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:46:46.289122417Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:47:33.920097986Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-------------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4a125bdffd7e2d867f706d4af5adbf4e7de36268dde119e47c6614cc7579317c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4465/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507978727919616","occurred_at":"2026-03-16T11:37:31Z","failed_at":"e2e","signature_id":"e9917fbf9b6bac9b4d087459a6fd771119a1e82c1b569a00e859383abfef9669","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e1c1e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2s8ksj\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000e27060\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-2s8ksj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e27040\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-2s8ksj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e27020\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-2s8ksj\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e1c1e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2s8ksj\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000e27060\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-2s8ksj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e27040\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-2s8ksj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e27020\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-2s8ksj\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"5b6757126b2455b8ef48ed3bdcb352d288978be1deefdd68a6598e68887b2497","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4465/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507978727919616","occurred_at":"2026-03-16T11:37:31Z","failed_at":"e2e","signature_id":"87ddebd77cb751c3309e54fd550cf1c2ee10ecad75b0b902674b283731b1c170","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:242]: Unexpected error:\n \u003c*errors.joinError | 0xc001101680\u003e: \n context deadline exceeded\n cluster operators not available: image-registry (Available=False, Progressing=True, Degraded=True)\n {\n errs: [\n \u003ccontext.deadlineExceededError\u003e{},\n \u003c*errors.errorString | 0xc0006ec9b0\u003e{\n s: \"cluster operators not available: image-registry (Available=False, Progressing=True, Degraded=True)\",\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:242]: Unexpected error:\n \u003c*errors.joinError | 0xc001101680\u003e: \n context deadline exceeded\n cluster operators not available: image-registry (Available=False, Progressing=True, Degraded=True)\n {\n errs: [\n \u003ccontext.deadlineExceededError\u003e{},\n \u003c*errors.errorString | 0xc0006ec9b0\u003e{\n s: \"cluster operators not available: image-registry (Available=False, Progressing=True, Degraded=True)\",\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2a050fd175f9ba873bc55c63e6fe807f7b435ab5054408cba89702d9e47ef41f","failure_pattern":"cluster operators not available: image-registry (Available=False, Progressing=True, Degraded=True)","search_query":"cluster operators not available: image-registry (Available=False, Progressing=True, Degraded=True)"}},{"row_id":"881149d7d9feb4396173694e36f2fcb9b6ce434f34c84e114b62d3f5289785af","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4465/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507978727919616","occurred_at":"2026-03-16T11:37:31Z","failed_at":"e2e","signature_id":"e17e5d5aa091e92b9bf8780d3c7be5775070b71e6a6ffb3deb697d8c25bde717","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00075d000\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-49g9c7\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5f7e5df7-d77c-4719-ab29-57d47b6ec43e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5f7e5df7-d77c-4719-ab29-57d47b6ec43e\",\n \"name\": \"5f7e5df7-d77c-4719-ab29-57d47b6ec43e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:32:46.202033733Z\",\n \"endTime\": \"2026-03-16T12:33:12.711755824Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-0/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-0/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-49g9c7\\\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5f7e5df7-d77c-4719-ab29-57d47b6ec43e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5f7e5df7-d77c-4719-ab29-57d47b6ec43e\\\",\\n \\\"name\\\": \\\"5f7e5df7-d77c-4719-ab29-57d47b6ec43e\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:32:46.202033733Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:33:12.711755824Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-0/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-0/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001488510\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Mon, 16 Mar 2026 12:33:16 GMT\",\n ],\n \"Content-Length\": [\"1561\"],\n \"X-Envoy-Upstream-Service-Time\": [\"57\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"e5d2fd0e-17e5-49ac-ac1c-60489cae0e3f\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00075c9c0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 53, 102, 55, 101, 53, 100, 102, 55, 45, 100, 55, 55, 99, 45, 52, 55, 49, 57, 45, 97, 98, 50, 57, 45, 53, 55, 100, 52, 55, 98, 54, 101, 99, 52, 51, 101, 34, 44, 10, 32, 32, 32, 32, 34, 11...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00075d000\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-49g9c7\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5f7e5df7-d77c-4719-ab29-57d47b6ec43e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5f7e5df7-d77c-4719-ab29-57d47b6ec43e\",\n \"name\": \"5f7e5df7-d77c-4719-ab29-57d47b6ec43e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:32:46.202033733Z\",\n \"endTime\": \"2026-03-16T12:33:12.711755824Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-0/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-0/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-49g9c7\\\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5f7e5df7-d77c-4719-ab29-57d47b6ec43e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5f7e5df7-d77c-4719-ab29-57d47b6ec43e\\\",\\n \\\"name\\\": \\\"5f7e5df7-d77c-4719-ab29-57d47b6ec43e\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:32:46.202033733Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:33:12.711755824Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-0/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-0/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001488510\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Mon, 16 Mar 2026 12:33:16 GMT\",\n ],\n \"Content-Length\": [\"1561\"],\n \"X-Envoy-Upstream-Service-Time\": [\"57\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"e5d2fd0e-17e5-49ac-ac1c-60489cae0e3f\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00075c9c0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 53, 102, 55, 101, 53, 100, 102, 55, 45, 100, 55, 55, 99, 45, 52, 55, 49, 57, 45, 97, 98, 50, 57, 45, 53, 55, 100, 52, 55, 98, 54, 101, 99, 52, 51, 101, 34, 44, 10, 32, 32, 32, 32, 34, 11...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"88a71154c9140a2dd26e175e4a7ae316646552a7b5ad5e62747429abad1a84a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4465/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507978727919616","occurred_at":"2026-03-16T11:37:31Z","failed_at":"e2e","signature_id":"d93e08e29ee9cb5bd3d7dc64d1e786a73a5feb7d81f5b67f709060a6dd09329a","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00105e300\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-ffm6zs\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001692f40\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-ffm6zs\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001692f20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-ffm6zs\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001692f00\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-ffm6zs\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00105e300\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-ffm6zs\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001692f40\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-ffm6zs\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001692f20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-ffm6zs\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001692f00\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-ffm6zs\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"906f3d64d0803bcbb4d625ae5ba14325688b4aa295e3250a05b630c57e88b526","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4465/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507978727919616","occurred_at":"2026-03-16T11:37:31Z","failed_at":"e2e","signature_id":"2973460e0ee4e2e7df18c463b4821794f2b68ee20e63198a4a63dd4efae12f8c","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0009de168\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-8cnmbs\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0008896e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-8cnmbs\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008896c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-8cnmbs\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008896a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-8cnmbs\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0009de168\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-8cnmbs\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0008896e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-8cnmbs\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008896c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-8cnmbs\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008896a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-8cnmbs\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"b1cfad9d99e4ca20d71baad5fdfa9cd25e0d92b6f6bf280d73c8d0d002607ab0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4465/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507978727919616","occurred_at":"2026-03-16T11:37:31Z","failed_at":"e2e","signature_id":"0539329b0e11aa75549429ce7fb0503fdf887abff5f93909bd303583c235a0b8","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:89]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000492960\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-6lzq82\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/559203f1-1420-4ab3-b051-c70ea7bf34e2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/559203f1-1420-4ab3-b051-c70ea7bf34e2\",\n \"name\": \"559203f1-1420-4ab3-b051-c70ea7bf34e2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:36:11.314175626Z\",\n \"endTime\": \"2026-03-16T12:37:05.885517874Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-0/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-0/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-6lzq82\\\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/559203f1-1420-4ab3-b051-c70ea7bf34e2\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/559203f1-1420-4ab3-b051-c70ea7bf34e2\\\",\\n \\\"name\\\": \\\"559203f1-1420-4ab3-b051-c70ea7bf34e2\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:36:11.314175626Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:37:05.885517874Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-0/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-0/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000492920\u003e{\n msg: \"failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-6lzq82\\\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/559203f1-1420-4ab3-b051-c70ea7bf34e2\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/559203f1-1420-4ab3-b051-c70ea7bf34e2\\\",\\n \\\"name\\\": \\\"559203f1-1420-4ab3-b051-c70ea7bf34e2\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:36:11.314175626Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:37:05.885517874Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-0/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-0/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:89]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000492960\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-6lzq82\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/559203f1-1420-4ab3-b051-c70ea7bf34e2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/559203f1-1420-4ab3-b051-c70ea7bf34e2\",\n \"name\": \"559203f1-1420-4ab3-b051-c70ea7bf34e2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:36:11.314175626Z\",\n \"endTime\": \"2026-03-16T12:37:05.885517874Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-0/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-0/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-6lzq82\\\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/559203f1-1420-4ab3-b051-c70ea7bf34e2\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/559203f1-1420-4ab3-b051-c70ea7bf34e2\\\",\\n \\\"name\\\": \\\"559203f1-1420-4ab3-b051-c70ea7bf34e2\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:36:11.314175626Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:37:05.885517874Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-0/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-0/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000492920\u003e{\n msg: \"failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-6lzq82\\\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/559203f1-1420-4ab3-b051-c70ea7bf34e2\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/559203f1-1420-4ab3-b051-c70ea7bf34e2\\\",\\n \\\"name\\\": \\\"559203f1-1420-4ab3-b051-c70ea7bf34e2\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:36:11.314175626Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:37:05.885517874Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-0/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-0/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c25cad12eb27560b4350c5cc4cd1bcf48b0a7508521f93ef01aa3af1bb5c0dba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4465/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507978727919616","occurred_at":"2026-03-16T11:37:31Z","failed_at":"e2e","signature_id":"e5735ba7f9d79740322bf452904c53abe61372e741f624fc94f5cd3ada8d6e6c","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00074a480\u003e: \n failed to create HCP cluster cluster-zstream-4-21-p8thq2: failed waiting for cluster=\"cluster-zstream-4-21-p8thq2\" in resourcegroup=\"rg-zstream-upgrade-4-21-kpc4hw\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e1cf903-ac74-4008-9844-616ceb275703\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e1cf903-ac74-4008-9844-616ceb275703\",\n \"name\": \"8e1cf903-ac74-4008-9844-616ceb275703\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:36:47.414774401Z\",\n \"endTime\": \"2026-03-16T12:38:15.590235552Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-21-p8thq2: failed waiting for cluster=\\\"cluster-zstream-4-21-p8thq2\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-kpc4hw\\\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e1cf903-ac74-4008-9844-616ceb275703\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e1cf903-ac74-4008-9844-616ceb275703\\\",\\n \\\"name\\\": \\\"8e1cf903-ac74-4008-9844-616ceb275703\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:36:47.414774401Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:38:15.590235552Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00074a460\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-p8thq2\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-kpc4hw\\\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e1cf903-ac74-4008-9844-616ceb275703\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e1cf903-ac74-4008-9844-616ceb275703\\\",\\n \\\"name\\\": \\\"8e1cf903-ac74-4008-9844-616ceb275703\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:36:47.414774401Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:38:15.590235552Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: Subscript...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00074a480\u003e: \n failed to create HCP cluster cluster-zstream-4-21-p8thq2: failed waiting for cluster=\"cluster-zstream-4-21-p8thq2\" in resourcegroup=\"rg-zstream-upgrade-4-21-kpc4hw\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e1cf903-ac74-4008-9844-616ceb275703\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e1cf903-ac74-4008-9844-616ceb275703\",\n \"name\": \"8e1cf903-ac74-4008-9844-616ceb275703\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:36:47.414774401Z\",\n \"endTime\": \"2026-03-16T12:38:15.590235552Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-21-p8thq2: failed waiting for cluster=\\\"cluster-zstream-4-21-p8thq2\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-kpc4hw\\\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e1cf903-ac74-4008-9844-616ceb275703\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e1cf903-ac74-4008-9844-616ceb275703\\\",\\n \\\"name\\\": \\\"8e1cf903-ac74-4008-9844-616ceb275703\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:36:47.414774401Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:38:15.590235552Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00074a460\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-p8thq2\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-kpc4hw\\\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e1cf903-ac74-4008-9844-616ceb275703\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e1cf903-ac74-4008-9844-616ceb275703\\\",\\n \\\"name\\\": \\\"8e1cf903-ac74-4008-9844-616ceb275703\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:36:47.414774401Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:38:15.590235552Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-81/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: Subscript...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d1c83dbba2069ee9d1d9e185d342c29575acbe827f4e9ac94b10dbbdf0a1b31f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4465/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507978727919616","occurred_at":"2026-03-16T11:37:31Z","failed_at":"e2e","signature_id":"78b5da59e59157cfa75fe34bf31e36e11bfab8b465baedb5bb6860bf01168fe6","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ed04b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listtjr4ps\" in resourcegroup=\"ea-list-6vpjv7\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00076bdc0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listtjr4ps\\\" in resourcegroup=\\\"ea-list-6vpjv7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00076bda0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listtjr4ps\\\" in resourcegroup=\\\"ea-list-6vpjv7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00076bd80\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listtjr4ps\\\" in resourcegroup=\\\"ea-list-6vpjv7\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ed04b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listtjr4ps\" in resourcegroup=\"ea-list-6vpjv7\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00076bdc0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listtjr4ps\\\" in resourcegroup=\\\"ea-list-6vpjv7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00076bda0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listtjr4ps\\\" in resourcegroup=\\\"ea-list-6vpjv7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00076bd80\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listtjr4ps\\\" in resourcegroup=\\\"ea-list-6vpjv7\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"ecc284634bd9f829f977842d59f93cb444ae57e3117d4e253bea2621bbde3fbf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4465/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507978727919616","occurred_at":"2026-03-16T11:37:31Z","failed_at":"e2e","signature_id":"f5710c9a36a79ec7cc99b5b9893d96f1b912945427435c538a6e0cf3f28f964a","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000821180\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-q8n88z\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73331c6d-bdfd-4ef6-85d5-b0272f79589b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73331c6d-bdfd-4ef6-85d5-b0272f79589b\",\n \"name\": \"73331c6d-bdfd-4ef6-85d5-b0272f79589b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:32:45.831342796Z\",\n \"endTime\": \"2026-03-16T12:33:12.654919141Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-q8n88z\\\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73331c6d-bdfd-4ef6-85d5-b0272f79589b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73331c6d-bdfd-4ef6-85d5-b0272f79589b\\\",\\n \\\"name\\\": \\\"73331c6d-bdfd-4ef6-85d5-b0272f79589b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:32:45.831342796Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:33:12.654919141Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000821160\u003e{\n msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-q8n88z\\\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73331c6d-bdfd-4ef6-85d5-b0272f79589b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73331c6d-bdfd-4ef6-85d5-b0272f79589b\\\",\\n \\\"name\\\": \\\"73331c6d-bdfd-4ef6-85d5-b0272f79589b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:32:45.831342796Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:33:12.654919141Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-----------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000821180\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-q8n88z\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73331c6d-bdfd-4ef6-85d5-b0272f79589b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73331c6d-bdfd-4ef6-85d5-b0272f79589b\",\n \"name\": \"73331c6d-bdfd-4ef6-85d5-b0272f79589b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:32:45.831342796Z\",\n \"endTime\": \"2026-03-16T12:33:12.654919141Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-q8n88z\\\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73331c6d-bdfd-4ef6-85d5-b0272f79589b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73331c6d-bdfd-4ef6-85d5-b0272f79589b\\\",\\n \\\"name\\\": \\\"73331c6d-bdfd-4ef6-85d5-b0272f79589b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:32:45.831342796Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:33:12.654919141Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000821160\u003e{\n msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-q8n88z\\\" to finish creating: GET https://rp.j7919616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73331c6d-bdfd-4ef6-85d5-b0272f79589b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73331c6d-bdfd-4ef6-85d5-b0272f79589b\\\",\\n \\\"name\\\": \\\"73331c6d-bdfd-4ef6-85d5-b0272f79589b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:32:45.831342796Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:33:12.654919141Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-----------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fbdc1d8aa0beb40535f015a0a132fa3123295310229468a80d53b638717816f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4465/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507978727919616","occurred_at":"2026-03-16T11:37:31Z","failed_at":"e2e","signature_id":"49d9fb57103771387a5d240c2047e450d2e7badfeebe6592cade8a1ba05cd55b","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d86f90\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-nn74pg\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000b0fa00\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-nn74pg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b0f980\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-nn74pg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b0f960\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-nn74pg\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d86f90\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-nn74pg\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000b0fa00\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-nn74pg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b0f980\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-nn74pg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b0f960\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-nn74pg\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":14,"failure_pattern_ids":["2a050fd175f9ba873bc55c63e6fe807f7b435ab5054408cba89702d9e47ef41f","503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761"],"matched_occurrences":14,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4466/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507921534390272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4466,"pr_state":"closed","pr_sha":"710411a37b0314ac490bc74bb1b808d11b92c9ff","final_merged_sha":"8cb4bd5f1213fd23ab6fcccc158048912ff9661e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:37:23Z"},"failed_at":["e2e"],"failed_test_count":10,"occurrences":[{"row_id":"0024ee1c4fe1abde385510dac409c46c5f2b7425b4a6a843ea13d88a4c79680c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4466/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507921534390272","occurred_at":"2026-03-16T11:37:23Z","failed_at":"e2e","signature_id":"9ad42114fa0d118ea2fea0feea7d08783218bdf66fec4e00d1dc55789717dad0","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0016d81e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-sr7fvl\" in resourcegroup=\"rg-zstream-upgrade-4-20-tfrc9m\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f90fe0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-sr7fvl\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tfrc9m\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f90fc0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-sr7fvl\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tfrc9m\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f90fa0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-20-sr7fvl\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tfrc9m\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0016d81e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-sr7fvl\" in resourcegroup=\"rg-zstream-upgrade-4-20-tfrc9m\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f90fe0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-sr7fvl\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tfrc9m\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f90fc0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-20-sr7fvl\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tfrc9m\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f90fa0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-20-sr7fvl\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tfrc9m\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"480ec8f06e5a57cdf70de85ded09cd02ba4f8515a4e796f47cbe872ca37b0375","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4466/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507921534390272","occurred_at":"2026-03-16T11:37:23Z","failed_at":"e2e","signature_id":"7c2a8ae211b240fbe18fa2136804cb0dfcd171339c8853482da7ac20c1395689","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010f8348\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-k8zshn\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0008d3ba0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-k8zshn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008d3b80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-k8zshn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008d3b60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-k8zshn\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010f8348\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-k8zshn\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0008d3ba0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-k8zshn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008d3b80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-k8zshn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008d3b60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-k8zshn\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"5067d088305f7a05477bcadbb603b6d0fe56db765916e0b7695b2aba7440485a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4466/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507921534390272","occurred_at":"2026-03-16T11:37:23Z","failed_at":"e2e","signature_id":"ddc900ad43e3307bb95227508a52f98d21f7ee6987ba7967c415911a2259a393","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0013c6180\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-x864pq\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000135440\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-x864pq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000135420\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-x864pq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0001353e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-x864pq\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0013c6180\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-x864pq\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000135440\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-x864pq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000135420\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-x864pq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0001353e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-x864pq\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"635c5ec217916f02633dd2a955002b4e6ebb102e90d5645697e39ed9dee1a1bb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4466/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507921534390272","occurred_at":"2026-03-16T11:37:23Z","failed_at":"e2e","signature_id":"68b2f0d5a7ca7f13b2f06651d2461c1b3acbf2ec813a90d3e32e21dd2912c8c5","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:86]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e6c4c0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-8svmbp\" to finish creating: GET https://rp.j4390272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73dcdb23-9553-4247-929b-fe1cac143a88\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73dcdb23-9553-4247-929b-fe1cac143a88\",\n \"name\": \"73dcdb23-9553-4247-929b-fe1cac143a88\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:49:19.684427225Z\",\n \"endTime\": \"2026-03-16T12:50:43.423575122Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-83/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-83/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-8svmbp\\\" to finish creating: GET https://rp.j4390272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73dcdb23-9553-4247-929b-fe1cac143a88\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73dcdb23-9553-4247-929b-fe1cac143a88\\\",\\n \\\"name\\\": \\\"73dcdb23-9553-4247-929b-fe1cac143a88\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:49:19.684427225Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:50:43.423575122Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-83/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-83/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e6c4a0\u003e{\n msg: \"failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-8svmbp\\\" to finish creating: GET https://rp.j4390272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73dcdb23-9553-4247-929b-fe1cac143a88\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73dcdb23-9553-4247-929b-fe1cac143a88\\\",\\n \\\"name\\\": \\\"73dcdb23-9553-4247-929b-fe1cac143a88\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:49:19.684427225Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:50:43.423575122Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-83/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-83/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n---------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:86]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e6c4c0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-8svmbp\" to finish creating: GET https://rp.j4390272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73dcdb23-9553-4247-929b-fe1cac143a88\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73dcdb23-9553-4247-929b-fe1cac143a88\",\n \"name\": \"73dcdb23-9553-4247-929b-fe1cac143a88\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:49:19.684427225Z\",\n \"endTime\": \"2026-03-16T12:50:43.423575122Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-83/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-83/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-8svmbp\\\" to finish creating: GET https://rp.j4390272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73dcdb23-9553-4247-929b-fe1cac143a88\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73dcdb23-9553-4247-929b-fe1cac143a88\\\",\\n \\\"name\\\": \\\"73dcdb23-9553-4247-929b-fe1cac143a88\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:49:19.684427225Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:50:43.423575122Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-83/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-83/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e6c4a0\u003e{\n msg: \"failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-8svmbp\\\" to finish creating: GET https://rp.j4390272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73dcdb23-9553-4247-929b-fe1cac143a88\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73dcdb23-9553-4247-929b-fe1cac143a88\\\",\\n \\\"name\\\": \\\"73dcdb23-9553-4247-929b-fe1cac143a88\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:49:19.684427225Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:50:43.423575122Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-83/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-83/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n---------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6fdf026a6b5bac300ffdc2011ed18c77f2f597520ddecfe9c41d56a1d06ca1fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4466/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507921534390272","occurred_at":"2026-03-16T11:37:23Z","failed_at":"e2e","signature_id":"082bce810adaa6468e51e21422b820cf5456889bca2de282de465cd2bd20c4bb","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0011ee2e8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-jwm7pz\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0009a9bc0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-jwm7pz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009a9ba0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-jwm7pz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009a9b00\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-jwm7pz\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0011ee2e8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-jwm7pz\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0009a9bc0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-jwm7pz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009a9ba0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-jwm7pz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009a9b00\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-jwm7pz\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"85757536c70e7fdf921a242127316ccd6ca91a32fe5756dad0c01eb0ee0faf60","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4466/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507921534390272","occurred_at":"2026-03-16T11:37:23Z","failed_at":"e2e","signature_id":"23512d541a958b440954c2ac1b7f62c1d8e5db78f2fa1096d20ef7698e405e75","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001340320\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-qd594h\" to finish creating: GET https://rp.j4390272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7f8e94da-9c36-4738-8107-1faf2b611256\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7f8e94da-9c36-4738-8107-1faf2b611256\",\n \"name\": \"7f8e94da-9c36-4738-8107-1faf2b611256\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:26:39.838085664Z\",\n \"endTime\": \"2026-03-16T12:27:14.55398244Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-228/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-228/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-qd594h\\\" to finish creating: GET https://rp.j4390272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7f8e94da-9c36-4738-8107-1faf2b611256\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7f8e94da-9c36-4738-8107-1faf2b611256\\\",\\n \\\"name\\\": \\\"7f8e94da-9c36-4738-8107-1faf2b611256\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:26:39.838085664Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:27:14.55398244Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-228/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-228/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001340300\u003e{\n msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-qd594h\\\" to finish creating: GET https://rp.j4390272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7f8e94da-9c36-4738-8107-1faf2b611256\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7f8e94da-9c36-4738-8107-1faf2b611256\\\",\\n \\\"name\\\": \\\"7f8e94da-9c36-4738-8107-1faf2b611256\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:26:39.838085664Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:27:14.55398244Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-228/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-228/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n----------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001340320\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-qd594h\" to finish creating: GET https://rp.j4390272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7f8e94da-9c36-4738-8107-1faf2b611256\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7f8e94da-9c36-4738-8107-1faf2b611256\",\n \"name\": \"7f8e94da-9c36-4738-8107-1faf2b611256\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:26:39.838085664Z\",\n \"endTime\": \"2026-03-16T12:27:14.55398244Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-228/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-228/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-qd594h\\\" to finish creating: GET https://rp.j4390272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7f8e94da-9c36-4738-8107-1faf2b611256\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7f8e94da-9c36-4738-8107-1faf2b611256\\\",\\n \\\"name\\\": \\\"7f8e94da-9c36-4738-8107-1faf2b611256\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:26:39.838085664Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:27:14.55398244Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-228/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-228/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001340300\u003e{\n msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-qd594h\\\" to finish creating: GET https://rp.j4390272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7f8e94da-9c36-4738-8107-1faf2b611256\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7f8e94da-9c36-4738-8107-1faf2b611256\\\",\\n \\\"name\\\": \\\"7f8e94da-9c36-4738-8107-1faf2b611256\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:26:39.838085664Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:27:14.55398244Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-228/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-228/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n----------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"aa9507604f5adc2da80b1ae6168ae03fae2743d5d52c7e09936c6fb6fd4cf6fc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4466/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507921534390272","occurred_at":"2026-03-16T11:37:23Z","failed_at":"e2e","signature_id":"8d546e2809a80d6c7efb3203a7cc6e2c5670fd721ccdcc8567a071fe193aca48","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001070330\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-fwdxl9\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00067a840\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fwdxl9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00067a820\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fwdxl9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00067a800\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fwdxl9\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001070330\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-fwdxl9\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00067a840\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fwdxl9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00067a820\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fwdxl9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00067a800\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fwdxl9\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"be9f9e4b95fceeeddb99e8858dfa7eba292463935c6b361df18516aace181e80","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4466/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507921534390272","occurred_at":"2026-03-16T11:37:23Z","failed_at":"e2e","signature_id":"eba0ed29f8aba0a1d37896c000c9d2b3c1327557a6caf7f107c40155eed28ba3","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0009aa330\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-kszlkh\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00167af40\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-kszlkh\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00167af20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-kszlkh\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00167af00\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-kszlkh\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0009aa330\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-kszlkh\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00167af40\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-kszlkh\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00167af20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-kszlkh\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00167af00\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-kszlkh\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"d7b05f8a5db63c119e77b9db506bc25c40550ff6c76c124ce6a7f62981d46aae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4466/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507921534390272","occurred_at":"2026-03-16T11:37:23Z","failed_at":"e2e","signature_id":"7c0f3c6edd51d224a75f7c34cfd74b79486b545d1e20c0772c494df3f93f9e5c","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00017a8a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-r6hhrs\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00013d580\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-r6hhrs\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00013d560\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-r6hhrs\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00013d540\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-r6hhrs\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00017a8a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-r6hhrs\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00013d580\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-r6hhrs\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00013d560\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-r6hhrs\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00013d540\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-r6hhrs\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"f9122f52b3bd895ef3acfedcae5350284434350c8f4684e57e325caf7ef85ade","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4466/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507921534390272","occurred_at":"2026-03-16T11:37:23Z","failed_at":"e2e","signature_id":"01e392de4111bd4e65ad2bbd6458971ce92d8888587d19c39d7bc8df8d989792","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0015e40a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-92hzhp\" in resourcegroup=\"rg-zstream-upgrade-4-21-6847r5\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000e94e00\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-92hzhp\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-6847r5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e94de0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-92hzhp\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-6847r5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e94dc0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-92hzhp\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-6847r5\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0015e40a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-92hzhp\" in resourcegroup=\"rg-zstream-upgrade-4-21-6847r5\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000e94e00\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-92hzhp\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-6847r5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e94de0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-92hzhp\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-6847r5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e94dc0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-92hzhp\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-6847r5\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":10,"failure_pattern_ids":["503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761"],"matched_occurrences":10,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507889917726720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4467,"pr_state":"closed","pr_sha":"4834b3b749099f5b743a5d85207f30a5c96a61d4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:37:16Z"},"failed_at":["e2e"],"failed_test_count":15,"occurrences":[{"row_id":"166bf1c469ea8f74c93d979af3b9a44f2b1f7ccc6aec51b648d86c5b587fb275","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507889917726720","occurred_at":"2026-03-16T11:37:16Z","failed_at":"e2e","signature_id":"fd215487f6d8858ec36bdb292cbc87a4e40a2a115620326b8553d17da96f16c8","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000edce58\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-cs5465\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0008c77a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-cs5465\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008c7780\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-cs5465\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008c7760\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-cs5465\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xae100d0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000edce58\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-cs5465\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0008c77a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-cs5465\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008c7780\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-cs5465\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008c7760\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-cs5465\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xae100d0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"2aa3eae0eaedfc64a96df781ed123859a65c7b8a31b81996153a8dff4570533d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507889917726720","occurred_at":"2026-03-16T11:37:16Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:152]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:152]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"2e76ffd810affb86db714dd6620b49c98791e0646acf6d69a0a742522d917115","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507889917726720","occurred_at":"2026-03-16T11:37:16Z","failed_at":"e2e","signature_id":"4112aaa153fc4ae5e69c728ff46043e12d9245c30319506bc5e945acfafe7eaf","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000d9c5c0\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-bh8k9w\" to finish creating: GET https://rp.j7726720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/507fce4f-c4c5-40f4-a7c1-39b34b540b30\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/507fce4f-c4c5-40f4-a7c1-39b34b540b30\",\n \"name\": \"507fce4f-c4c5-40f4-a7c1-39b34b540b30\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:29:19.556311137Z\",\n \"endTime\": \"2026-03-16T14:30:55.368968887Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-255/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-255/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: 429\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"429\\\",\\n \\\"message\\\": \\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-bh8k9w\\\" to finish creating: GET https://rp.j7726720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/507fce4f-c4c5-40f4-a7c1-39b34b540b30\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/507fce4f-c4c5-40f4-a7c1-39b34b540b30\\\",\\n \\\"name\\\": \\\"507fce4f-c4c5-40f4-a7c1-39b34b540b30\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:29:19.556311137Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:30:55.368968887Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-255/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-255/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: 429\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"429\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001488870\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"070079fb-42c9-4860-a33d-946a989c0339\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 14:31:00 GMT\",\n ],\n \"Content-Length\": [\"1396\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000d9c3a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 53, 48, 55, 102, 99, 101, 52, 102, 45, 99, 52, 99, 53, 45, 52, 48, 102, 52, 45, 97, 55, 99, 49, 45, 51, 57, 98, 51, 52, 98, 53, 52, 48, 98, 51, 48, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 53, 48, 55, 102, 99, 101, 52, 102, 45, 99, 52, 99, 53, 45, 52, 48, 102, 52, 45, 97, 55, 99, 49, 45, 51, 57, 98, 51, 52, 98, 53, 52, 48, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000d9c5c0\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-bh8k9w\" to finish creating: GET https://rp.j7726720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/507fce4f-c4c5-40f4-a7c1-39b34b540b30\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/507fce4f-c4c5-40f4-a7c1-39b34b540b30\",\n \"name\": \"507fce4f-c4c5-40f4-a7c1-39b34b540b30\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:29:19.556311137Z\",\n \"endTime\": \"2026-03-16T14:30:55.368968887Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-255/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-255/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: 429\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"429\\\",\\n \\\"message\\\": \\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-bh8k9w\\\" to finish creating: GET https://rp.j7726720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/507fce4f-c4c5-40f4-a7c1-39b34b540b30\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/507fce4f-c4c5-40f4-a7c1-39b34b540b30\\\",\\n \\\"name\\\": \\\"507fce4f-c4c5-40f4-a7c1-39b34b540b30\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:29:19.556311137Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:30:55.368968887Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-255/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-255/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: 429\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"429\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001488870\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"070079fb-42c9-4860-a33d-946a989c0339\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 14:31:00 GMT\",\n ],\n \"Content-Length\": [\"1396\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000d9c3a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 53, 48, 55, 102, 99, 101, 52, 102, 45, 99, 52, 99, 53, 45, 52, 48, 102, 52, 45, 97, 55, 99, 49, 45, 51, 57, 98, 51, 52, 98, 53, 52, 48, 98, 51, 48, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 53, 48, 55, 102, 99, 101, 52, 102, 45, 99, 52, 99, 53, 45, 52, 48, 102, 52, 45, 97, 55, 99, 49, 45, 51, 57, 98, 51, 52, 98, 53, 52, 48, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4ea33acd3f1140f6ce3e5704a686a67d6df1f6b9e581d4685b990aa2749d6360","failure_pattern":"ERROR CODE: InternalServerError; detail code 429; detail message Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"32773a5eead6653db48b51131308635da8bfd013324f3b076699fe13bfd120cb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507889917726720","occurred_at":"2026-03-16T11:37:16Z","failed_at":"e2e","signature_id":"55f20db013a62e2bc66124fd46e741977748eb70c8a98a5bff11b84d5a64643a","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0009f5520\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-r4548l\" to finish creating: GET https://rp.j7726720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7821c08d-3139-459a-921e-3f15f32a1968\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7821c08d-3139-459a-921e-3f15f32a1968\",\n \"name\": \"7821c08d-3139-459a-921e-3f15f32a1968\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:29:29.983068909Z\",\n \"endTime\": \"2026-03-16T14:30:55.403589972Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-155/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-155/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: 429\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"429\\\",\\n \\\"message\\\": \\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 2 seconds.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster ea-cluster: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-r4548l\\\" to finish creating: GET https://rp.j7726720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7821c08d-3139-459a-921e-3f15f32a1968\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7821c08d-3139-459a-921e-3f15f32a1968\\\",\\n \\\"name\\\": \\\"7821c08d-3139-459a-921e-3f15f32a1968\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:29:29.983068909Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:30:55.403589972Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-155/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-155/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: 429\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"429\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 2 seconds.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0009f5500\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-r4548l\\\" to finish creating: GET https://rp.j7726720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7821c08d-3139-459a-921e-3f15f32a1968\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7821c08d-3139-459a-921e-3f15f32a1968\\\",\\n \\\"name\\\": \\\"7821c08d-3139-459a-921e-3f15f32a1968\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:29:29.983068909Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:30:55.403589972Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-155/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-155/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: 429\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"429\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued t...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0009f5520\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-r4548l\" to finish creating: GET https://rp.j7726720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7821c08d-3139-459a-921e-3f15f32a1968\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7821c08d-3139-459a-921e-3f15f32a1968\",\n \"name\": \"7821c08d-3139-459a-921e-3f15f32a1968\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:29:29.983068909Z\",\n \"endTime\": \"2026-03-16T14:30:55.403589972Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-155/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-155/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: 429\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"429\\\",\\n \\\"message\\\": \\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 2 seconds.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster ea-cluster: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-r4548l\\\" to finish creating: GET https://rp.j7726720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7821c08d-3139-459a-921e-3f15f32a1968\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7821c08d-3139-459a-921e-3f15f32a1968\\\",\\n \\\"name\\\": \\\"7821c08d-3139-459a-921e-3f15f32a1968\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:29:29.983068909Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:30:55.403589972Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-155/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-155/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: 429\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"429\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 2 seconds.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0009f5500\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-r4548l\\\" to finish creating: GET https://rp.j7726720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7821c08d-3139-459a-921e-3f15f32a1968\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7821c08d-3139-459a-921e-3f15f32a1968\\\",\\n \\\"name\\\": \\\"7821c08d-3139-459a-921e-3f15f32a1968\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:29:29.983068909Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:30:55.403589972Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-155/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-155/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: 429\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"429\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued t...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"30fa2fa857c42c0b459f74a2bbc1050bfe224ec724cca8fb5c955cd9abb6b04a","failure_pattern":"ERROR CODE: InternalServerError; detail code 429; detail message Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 2 seconds.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4c402ad94268c513391f91a21d9d7104dc6e074ae170199f85ed73800ce153dd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507889917726720","occurred_at":"2026-03-16T11:37:16Z","failed_at":"e2e","signature_id":"e4dc9c08e593e9ca466e02ecd53f4127df97ddacf51da0ded82cf72f55d252c3","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0016aa300\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-glhwtw\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a21880\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-glhwtw\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a21860\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-glhwtw\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a21840\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-glhwtw\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xae100d0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0016aa300\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-glhwtw\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a21880\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-glhwtw\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a21860\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-glhwtw\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a21840\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-glhwtw\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xae100d0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"51d353aac044d0a477e4578b3c9b67a7a65abd298da2b6a71d7e99d726229159","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507889917726720","occurred_at":"2026-03-16T11:37:16Z","failed_at":"e2e","signature_id":"fdb96c69f4a4c1ceda26a7ede8b7831c9980894df0ba950b90fb9169b234f9ff","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00127c060\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-7bhc2w\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0006c5ae0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-7bhc2w\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006c5aa0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-7bhc2w\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006c5a80\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-7bhc2w\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xae100d0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00127c060\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-7bhc2w\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0006c5ae0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-7bhc2w\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006c5aa0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-7bhc2w\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006c5a80\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-7bhc2w\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xae100d0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"5feff105be279a09a858914701c815263c7f38784e578e51d57552fe3f3e15bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507889917726720","occurred_at":"2026-03-16T11:37:16Z","failed_at":"e2e","signature_id":"6624c456e8b183fd57f57c2074915523ffc44882f152aab6509802be631be546","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000c10640\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-6qd28n\" to finish creating: GET https://rp.j7726720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ddb4a419-8db1-4ddb-8221-c56bc2816e59\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ddb4a419-8db1-4ddb-8221-c56bc2816e59\",\n \"name\": \"ddb4a419-8db1-4ddb-8221-c56bc2816e59\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:29:32.981550365Z\",\n \"endTime\": \"2026-03-16T14:30:55.380145231Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-161/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-161/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: 429\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"429\\\",\\n \\\"message\\\": \\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-6qd28n\\\" to finish creating: GET https://rp.j7726720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ddb4a419-8db1-4ddb-8221-c56bc2816e59\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ddb4a419-8db1-4ddb-8221-c56bc2816e59\\\",\\n \\\"name\\\": \\\"ddb4a419-8db1-4ddb-8221-c56bc2816e59\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:29:32.981550365Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:30:55.380145231Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-161/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-161/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: 429\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"429\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000c10620\u003e{\n msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-6qd28n\\\" to finish creating: GET https://rp.j7726720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ddb4a419-8db1-4ddb-8221-c56bc2816e59\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ddb4a419-8db1-4ddb-8221-c56bc2816e59\\\",\\n \\\"name\\\": \\\"ddb4a419-8db1-4ddb-8221-c56bc2816e59\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:29:32.981550365Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:30:55.380145231Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-161/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-161/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: 429\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"429\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Request is temporarily throttled because subscription 974ebd46-8ad3-41e3-afef-7...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000c10640\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-6qd28n\" to finish creating: GET https://rp.j7726720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ddb4a419-8db1-4ddb-8221-c56bc2816e59\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ddb4a419-8db1-4ddb-8221-c56bc2816e59\",\n \"name\": \"ddb4a419-8db1-4ddb-8221-c56bc2816e59\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:29:32.981550365Z\",\n \"endTime\": \"2026-03-16T14:30:55.380145231Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-161/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-161/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: 429\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"429\\\",\\n \\\"message\\\": \\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-6qd28n\\\" to finish creating: GET https://rp.j7726720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ddb4a419-8db1-4ddb-8221-c56bc2816e59\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ddb4a419-8db1-4ddb-8221-c56bc2816e59\\\",\\n \\\"name\\\": \\\"ddb4a419-8db1-4ddb-8221-c56bc2816e59\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:29:32.981550365Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:30:55.380145231Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-161/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-161/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: 429\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"429\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000c10620\u003e{\n msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-6qd28n\\\" to finish creating: GET https://rp.j7726720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ddb4a419-8db1-4ddb-8221-c56bc2816e59\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ddb4a419-8db1-4ddb-8221-c56bc2816e59\\\",\\n \\\"name\\\": \\\"ddb4a419-8db1-4ddb-8221-c56bc2816e59\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:29:32.981550365Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:30:55.380145231Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-161/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-161/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: 429\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"429\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Request is temporarily throttled because subscription 974ebd46-8ad3-41e3-afef-7...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4ea33acd3f1140f6ce3e5704a686a67d6df1f6b9e581d4685b990aa2749d6360","failure_pattern":"ERROR CODE: InternalServerError; detail code 429; detail message Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"719b45825d40c1b8bbc9ff9b9ee67a567ea7e41c84f8245c2fd6ac6a9227248e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507889917726720","occurred_at":"2026-03-16T11:37:16Z","failed_at":"e2e","signature_id":"7c18e4deb83cf3afabe3857c9289e9e1e2144e859e698f781cbb1898d7f95cb1","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0016ac138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-w8fwbm\" in resourcegroup=\"rg-zstream-upgrade-4-21-4p9fwg\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000e0b7e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-w8fwbm\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-4p9fwg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e0b7c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-w8fwbm\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-4p9fwg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e0b7a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-w8fwbm\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-4p9fwg\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xae100d0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0016ac138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-w8fwbm\" in resourcegroup=\"rg-zstream-upgrade-4-21-4p9fwg\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000e0b7e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-w8fwbm\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-4p9fwg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e0b7c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-21-w8fwbm\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-4p9fwg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e0b7a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-w8fwbm\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-4p9fwg\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xae100d0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"81678c91d0537e7c0d636f2a02203f72335290eee54ff2e70e00ecab32841756","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507889917726720","occurred_at":"2026-03-16T11:37:16Z","failed_at":"e2e","signature_id":"6319b3c9371ade0c0d4eac85be2c30a4d8037fc4f040fe8e61c3ba6228e0d61a","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001434a80\u003e: \n failed to create HCP cluster cluster-zstream-4-20-vxx48m: failed waiting for cluster=\"cluster-zstream-4-20-vxx48m\" in resourcegroup=\"rg-zstream-upgrade-4-20-px5chb\" to finish creating: GET https://rp.j7726720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e8e1de0-accf-4251-9554-eef8a21cb746\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e8e1de0-accf-4251-9554-eef8a21cb746\",\n \"name\": \"4e8e1de0-accf-4251-9554-eef8a21cb746\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:34:51.663043024Z\",\n \"endTime\": \"2026-03-16T14:36:01.013234838Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-255/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-255/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-20-vxx48m: failed waiting for cluster=\\\"cluster-zstream-4-20-vxx48m\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-px5chb\\\" to finish creating: GET https://rp.j7726720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e8e1de0-accf-4251-9554-eef8a21cb746\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e8e1de0-accf-4251-9554-eef8a21cb746\\\",\\n \\\"name\\\": \\\"4e8e1de0-accf-4251-9554-eef8a21cb746\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:34:51.663043024Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:36:01.013234838Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-255/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-255/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001434a60\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-vxx48m\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-px5chb\\\" to finish creating: GET https://rp.j7726720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e8e1de0-accf-4251-9554-eef8a21cb746\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e8e1de0-accf-4251-9554-eef8a21cb746\\\",\\n \\\"name\\\": \\\"4e8e1de0-accf-4251-9554-eef8a21cb746\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:34:51.663043024Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:36:01.013234838Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-255/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-255/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n---------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001434a80\u003e: \n failed to create HCP cluster cluster-zstream-4-20-vxx48m: failed waiting for cluster=\"cluster-zstream-4-20-vxx48m\" in resourcegroup=\"rg-zstream-upgrade-4-20-px5chb\" to finish creating: GET https://rp.j7726720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e8e1de0-accf-4251-9554-eef8a21cb746\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e8e1de0-accf-4251-9554-eef8a21cb746\",\n \"name\": \"4e8e1de0-accf-4251-9554-eef8a21cb746\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T14:34:51.663043024Z\",\n \"endTime\": \"2026-03-16T14:36:01.013234838Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-255/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-255/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-20-vxx48m: failed waiting for cluster=\\\"cluster-zstream-4-20-vxx48m\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-px5chb\\\" to finish creating: GET https://rp.j7726720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e8e1de0-accf-4251-9554-eef8a21cb746\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e8e1de0-accf-4251-9554-eef8a21cb746\\\",\\n \\\"name\\\": \\\"4e8e1de0-accf-4251-9554-eef8a21cb746\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:34:51.663043024Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:36:01.013234838Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-255/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-255/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001434a60\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-vxx48m\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-px5chb\\\" to finish creating: GET https://rp.j7726720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e8e1de0-accf-4251-9554-eef8a21cb746\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e8e1de0-accf-4251-9554-eef8a21cb746\\\",\\n \\\"name\\\": \\\"4e8e1de0-accf-4251-9554-eef8a21cb746\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T14:34:51.663043024Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T14:36:01.013234838Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-255/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-255/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n---------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8fb6d351745a8a3382e118445722fd26c64ae8f05c3d33fe81e0f4ec8022a0ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507889917726720","occurred_at":"2026-03-16T11:37:16Z","failed_at":"e2e","signature_id":"1c62939b246046a5fef5f117623ee187d5e5ca5a38e32beaddce98338c1aa6e0","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e4c288\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-jlbmd5\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0016b4ea0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-jlbmd5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0016b4e80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-jlbmd5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0016b4e60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-jlbmd5\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xae100d0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e4c288\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-jlbmd5\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0016b4ea0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-jlbmd5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0016b4e80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-jlbmd5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0016b4e60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-jlbmd5\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xae100d0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"977465b62596fd5e860043d4bc3df681c8a53964b127f8a87494fb1cb71ef50c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507889917726720","occurred_at":"2026-03-16T11:37:16Z","failed_at":"e2e","signature_id":"658bd8487329ee8e2229f29728c8c8d9185db4886bed6670982b1e4d3766767e","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.004s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000ecb3d0\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.004s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000ecb3d0\u003e: \n missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}},{"row_id":"a2db816dc3c1282af8c40716669d22e917283b464bdcf5c3a4c019eacd5f5435","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507889917726720","occurred_at":"2026-03-16T11:37:16Z","failed_at":"e2e","signature_id":"afa9db0025f9f96068d96e3a07f68c5f898afea4db8703d1a7523b878686f66f","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f3a168\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-tcvs9s\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007c8f20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-tcvs9s\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007c8f00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-tcvs9s\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007c8ee0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-tcvs9s\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xae100d0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f3a168\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-tcvs9s\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007c8f20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-tcvs9s\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007c8f00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-tcvs9s\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007c8ee0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-tcvs9s\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xae100d0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"c3e5108bccbbc12ab856c7bace27e4bd054e2617104e75145d51215844d00bb4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507889917726720","occurred_at":"2026-03-16T11:37:16Z","failed_at":"e2e","signature_id":"ff278e05fadee11109601db9658838b96820f85cdef91db4aca97a9c91f7e858","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000dee4c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-spbt5k\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0008fe700\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-spbt5k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008fe6e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-spbt5k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e0ee20\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-spbt5k\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xae100d0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000dee4c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-spbt5k\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0008fe700\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-spbt5k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0008fe6e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-spbt5k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000e0ee20\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-spbt5k\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xae100d0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"c85a576580181b78d6dfbb767a510d2af98470a8736421407f14ccb8c5f5492a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507889917726720","occurred_at":"2026-03-16T11:37:16Z","failed_at":"e2e","signature_id":"534d61738d3fb4d6cc97f0b4830390f5591cc073645e2810a1a90d00e1b55447","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00140e120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-24h8dp\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001490da0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-24h8dp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001490d80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-24h8dp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001490d60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-24h8dp\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xae100d0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00140e120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-24h8dp\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001490da0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-24h8dp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001490d80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-24h8dp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001490d60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-24h8dp\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xae100d0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"f420aac1ad8eadaa40f54dfe4c439c823bd38bc801d98325afd2fd0b55320349","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507889917726720","occurred_at":"2026-03-16T11:37:16Z","failed_at":"e2e","signature_id":"f14dc498a32e3fdc34060d5cc0b12f011654988cc35aab93feddec33d7bb8e8c","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0000130c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-mqv2gt\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000719300\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-mqv2gt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007192e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-mqv2gt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007192a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-mqv2gt\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xae100d0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0000130c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-mqv2gt\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000719300\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-mqv2gt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007192e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-mqv2gt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007192a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-mqv2gt\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xae100d0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":15,"failure_pattern_ids":["30fa2fa857c42c0b459f74a2bbc1050bfe224ec724cca8fb5c955cd9abb6b04a","4ea33acd3f1140f6ce3e5704a686a67d6df1f6b9e581d4685b990aa2749d6360","503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e"],"matched_occurrences":15,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507858162651136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4468,"pr_state":"closed","pr_sha":"0ffc4b6e13e4d2e627eb1e4f86ccb888dc1e7634","final_merged_sha":"532823b3d4dd565d5b0521f892350ccadb788955","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:37:08Z"},"failed_at":["e2e"],"failed_test_count":15,"occurrences":[{"row_id":"2f20fdcda1c49b4faf5e5557f3308e13e798750a653778070d01cde3727e9e22","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507858162651136","occurred_at":"2026-03-16T11:37:08Z","failed_at":"e2e","signature_id":"39d74154a7c0f7db52e12e700748e5f75fbca44a0cbeca6665eb07c5558b53c6","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00149c150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-w2cn9m\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000703960\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-w2cn9m\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000703900\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-w2cn9m\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007038e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-w2cn9m\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00149c150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-w2cn9m\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000703960\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-w2cn9m\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000703900\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-w2cn9m\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007038e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-w2cn9m\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"313a04f1b845dce540f6bd2059092cf28a5518a1e3362e16d165a734734b5701","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507858162651136","occurred_at":"2026-03-16T11:37:08Z","failed_at":"e2e","signature_id":"3f2a50d1228ce95ffeb92683d92d467f83726559499a19275e15acfdfb6b13d0","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000ebece0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-ptdfmd\" to finish creating: GET https://rp.j2651136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb5598e6-3412-4c1e-88db-7cb59ed1e7eb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb5598e6-3412-4c1e-88db-7cb59ed1e7eb\",\n \"name\": \"cb5598e6-3412-4c1e-88db-7cb59ed1e7eb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:53:51.06924507Z\",\n \"endTime\": \"2026-03-16T12:54:44.068464049Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-ptdfmd\\\" to finish creating: GET https://rp.j2651136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb5598e6-3412-4c1e-88db-7cb59ed1e7eb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb5598e6-3412-4c1e-88db-7cb59ed1e7eb\\\",\\n \\\"name\\\": \\\"cb5598e6-3412-4c1e-88db-7cb59ed1e7eb\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:53:51.06924507Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:54:44.068464049Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000ebecc0\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-ptdfmd\\\" to finish creating: GET https://rp.j2651136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb5598e6-3412-4c1e-88db-7cb59ed1e7eb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb5598e6-3412-4c1e-88db-7cb59ed1e7eb\\\",\\n \\\"name\\\": \\\"cb5598e6-3412-4c1e-88db-7cb59ed1e7eb\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:53:51.06924507Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:54:44.068464049Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n----------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000ebece0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-ptdfmd\" to finish creating: GET https://rp.j2651136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb5598e6-3412-4c1e-88db-7cb59ed1e7eb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb5598e6-3412-4c1e-88db-7cb59ed1e7eb\",\n \"name\": \"cb5598e6-3412-4c1e-88db-7cb59ed1e7eb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:53:51.06924507Z\",\n \"endTime\": \"2026-03-16T12:54:44.068464049Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-ptdfmd\\\" to finish creating: GET https://rp.j2651136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb5598e6-3412-4c1e-88db-7cb59ed1e7eb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb5598e6-3412-4c1e-88db-7cb59ed1e7eb\\\",\\n \\\"name\\\": \\\"cb5598e6-3412-4c1e-88db-7cb59ed1e7eb\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:53:51.06924507Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:54:44.068464049Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000ebecc0\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-ptdfmd\\\" to finish creating: GET https://rp.j2651136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb5598e6-3412-4c1e-88db-7cb59ed1e7eb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb5598e6-3412-4c1e-88db-7cb59ed1e7eb\\\",\\n \\\"name\\\": \\\"cb5598e6-3412-4c1e-88db-7cb59ed1e7eb\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:53:51.06924507Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:54:44.068464049Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n----------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4499a5030bf4bb8e532ce36131b7697834718467cd710a4614c45ed0fdbf3a0c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507858162651136","occurred_at":"2026-03-16T11:37:08Z","failed_at":"e2e","signature_id":"3ba7a144057bf84d5d48ca3576634d94f30ae9700963017410e6f461b44e2f10","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:242]: Unexpected error:\n \u003c*errors.joinError | 0xc000d9b548\u003e: \n context deadline exceeded\n failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\n {\n errs: [\n \u003ccontext.deadlineExceededError\u003e{},\n \u003c*fmt.wrapError | 0xc000492760\u003e{\n msg: \"failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xc000492740\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:242]: Unexpected error:\n \u003c*errors.joinError | 0xc000d9b548\u003e: \n context deadline exceeded\n failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\n {\n errs: [\n \u003ccontext.deadlineExceededError\u003e{},\n \u003c*fmt.wrapError | 0xc000492760\u003e{\n msg: \"failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xc000492740\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"51df8818945eaac2db384458dad7fe18e132e2957f792c66d932f6bdc8238c94","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507858162651136","occurred_at":"2026-03-16T11:37:08Z","failed_at":"e2e","signature_id":"42d8ebd1a54e31b8911a41b1c425e0b0861125b8d9b18311c0595fb3603d6f80","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00136e120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-zpjlr5\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001011580\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-zpjlr5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001011560\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-zpjlr5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001011540\u003e{\n msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-zpjlr5\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00136e120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-zpjlr5\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001011580\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-zpjlr5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001011560\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-zpjlr5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001011540\u003e{\n msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-zpjlr5\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"5d4a019de0de9302c372fe511174b2b1154c6acaa34ca182f451abbfcb94874c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507858162651136","occurred_at":"2026-03-16T11:37:08Z","failed_at":"e2e","signature_id":"339ded798f0ee824371125b36d3920eea15403212dadcae82ee7fc7912ad2074","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00116c498\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-7pvp62\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00065a660\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-7pvp62\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00065a640\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-7pvp62\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f8c7c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-7pvp62\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00116c498\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-7pvp62\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00065a660\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-7pvp62\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00065a640\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-7pvp62\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f8c7c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-7pvp62\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"5dfad16b201902e0f1d73f1f4ba48a668e9a78303cb614349aff796354c3da43","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507858162651136","occurred_at":"2026-03-16T11:37:08Z","failed_at":"e2e","signature_id":"397ffe2173d4a2f12ec1adfb49f8594b7e44744c307ccb9ea88ae977d6fe43ce","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0009c0340\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-kb7d8n\" to finish creating: GET https://rp.j2651136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/185fa830-1f51-4ccd-83ef-db03686add8d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/185fa830-1f51-4ccd-83ef-db03686add8d\",\n \"name\": \"185fa830-1f51-4ccd-83ef-db03686add8d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:49:01.211413542Z\",\n \"endTime\": \"2026-03-16T12:50:00.809428459Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-kb7d8n\\\" to finish creating: GET https://rp.j2651136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/185fa830-1f51-4ccd-83ef-db03686add8d\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/185fa830-1f51-4ccd-83ef-db03686add8d\\\",\\n \\\"name\\\": \\\"185fa830-1f51-4ccd-83ef-db03686add8d\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:49:01.211413542Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:50:00.809428459Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0009c0320\u003e{\n msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-kb7d8n\\\" to finish creating: GET https://rp.j2651136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/185fa830-1f51-4ccd-83ef-db03686add8d\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/185fa830-1f51-4ccd-83ef-db03686add8d\\\",\\n \\\"name\\\": \\\"185fa830-1f51-4ccd-83ef-db03686add8d\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:49:01.211413542Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:50:00.809428459Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0009c0340\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-kb7d8n\" to finish creating: GET https://rp.j2651136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/185fa830-1f51-4ccd-83ef-db03686add8d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/185fa830-1f51-4ccd-83ef-db03686add8d\",\n \"name\": \"185fa830-1f51-4ccd-83ef-db03686add8d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:49:01.211413542Z\",\n \"endTime\": \"2026-03-16T12:50:00.809428459Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-kb7d8n\\\" to finish creating: GET https://rp.j2651136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/185fa830-1f51-4ccd-83ef-db03686add8d\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/185fa830-1f51-4ccd-83ef-db03686add8d\\\",\\n \\\"name\\\": \\\"185fa830-1f51-4ccd-83ef-db03686add8d\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:49:01.211413542Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:50:00.809428459Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0009c0320\u003e{\n msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-kb7d8n\\\" to finish creating: GET https://rp.j2651136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/185fa830-1f51-4ccd-83ef-db03686add8d\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/185fa830-1f51-4ccd-83ef-db03686add8d\\\",\\n \\\"name\\\": \\\"185fa830-1f51-4ccd-83ef-db03686add8d\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:49:01.211413542Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:50:00.809428459Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-66/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"76caeb0afab332ea9145b553bdabe66e972598ba22e335f172d3204d1d1c021d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507858162651136","occurred_at":"2026-03-16T11:37:08Z","failed_at":"e2e","signature_id":"8787a34531aa9bdc19e3de4187bbf545c0eeeb95a6e8bd18b2a89e5d4fc1767f","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0014281f8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listm9jkb8\" in resourcegroup=\"ea-list-255dgx\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00140d080\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listm9jkb8\\\" in resourcegroup=\\\"ea-list-255dgx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00140d060\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listm9jkb8\\\" in resourcegroup=\\\"ea-list-255dgx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00140d040\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listm9jkb8\\\" in resourcegroup=\\\"ea-list-255dgx\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0014281f8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listm9jkb8\" in resourcegroup=\"ea-list-255dgx\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00140d080\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listm9jkb8\\\" in resourcegroup=\\\"ea-list-255dgx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00140d060\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listm9jkb8\\\" in resourcegroup=\\\"ea-list-255dgx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00140d040\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listm9jkb8\\\" in resourcegroup=\\\"ea-list-255dgx\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"82c4a0362327d086b389a974999b8f6c770987b8da279ecff4b88cad8e48efa1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507858162651136","occurred_at":"2026-03-16T11:37:08Z","failed_at":"e2e","signature_id":"4b340e46813538d4950bd3eff1dc561dbd6b46045b173c52913a00794bb6eaa1","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001287180\u003e: \n missing expected log sources: [ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [ocm-arohcp/kube-apiserver]\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001287180\u003e: \n missing expected log sources: [ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [ocm-arohcp/kube-apiserver]\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d68a3a4000afced929cc779e7a36af528e5b7a9739bb3b6369253f0a6d739eb2","failure_pattern":"missing expected log sources: [ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [ocm-arohcp/kube-apiserver]"}},{"row_id":"845b90730d85344c71c6fe2d1afd6ced5f43aa358b7d4593e37a332ff86df21c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507858162651136","occurred_at":"2026-03-16T11:37:08Z","failed_at":"e2e","signature_id":"9c1a56de853be0d64b8d30654c93d1b1174d1c3f179d705d12478a6c4b0d93f8","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0014641e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gqtm95\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000800040\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-gqtm95\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000800020\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-gqtm95\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007c1f60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-gqtm95\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0014641e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gqtm95\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000800040\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-gqtm95\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000800020\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-gqtm95\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007c1f60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-gqtm95\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"8f2106c4767bea2630770963cf58a71baa85c196b5cf7e53c3f6425c46f6e74f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507858162651136","occurred_at":"2026-03-16T11:37:08Z","failed_at":"e2e","signature_id":"d6df4085b3a2446ccc03d00bc02f85c6122e53b0d9a8c455f04fed3d202279e1","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001048300\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-pvkkz9\" in resourcegroup=\"admin-credential-lifecycle-test-t5rxqt\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00084b400\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-pvkkz9\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-t5rxqt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00084b3e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-pvkkz9\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-t5rxqt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00084b3c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-pvkkz9\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-t5rxqt\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001048300\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-pvkkz9\" in resourcegroup=\"admin-credential-lifecycle-test-t5rxqt\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00084b400\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-pvkkz9\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-t5rxqt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00084b3e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-pvkkz9\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-t5rxqt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00084b3c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-pvkkz9\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-t5rxqt\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"a31de6745ed6416a9d5aaee37f75ee4205bee600e64330cf095c7c75577b68ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507858162651136","occurred_at":"2026-03-16T11:37:08Z","failed_at":"e2e","signature_id":"38e1ebf666fdc35261844fa78df5ac62b98c08074ab133c3cc41e23a317084e9","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0014481b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-tk2rlb\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001500f60\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-tk2rlb\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001500f40\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-tk2rlb\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001500f20\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-tk2rlb\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0014481b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-tk2rlb\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001500f60\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-tk2rlb\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001500f40\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-tk2rlb\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001500f20\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-tk2rlb\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"abe55415eb51baca9c633d01b26835a004f9a608bccebb66579bddf3565d95b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507858162651136","occurred_at":"2026-03-16T11:37:08Z","failed_at":"e2e","signature_id":"0110eb7962a5daefe90f92403df2cf23641dd93befd355603355c3c04c6b4bf7","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0009ed3b0\u003e: \n failed to cleanup resource group: failed waiting for resourcegroup=\"clusternp128-4dtxfh\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0000deb20\u003e{\n msg: \"failed to cleanup resource group: failed waiting for resourcegroup=\\\"clusternp128-4dtxfh\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0000deb00\u003e{\n msg: \"failed waiting for resourcegroup=\\\"clusternp128-4dtxfh\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0009ed3b0\u003e: \n failed to cleanup resource group: failed waiting for resourcegroup=\"clusternp128-4dtxfh\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0000deb20\u003e{\n msg: \"failed to cleanup resource group: failed waiting for resourcegroup=\\\"clusternp128-4dtxfh\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0000deb00\u003e{\n msg: \"failed waiting for resourcegroup=\\\"clusternp128-4dtxfh\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d55c4a29bbbf19a43335bfa2157018ebc82a1c8fe8cbea22f78ef2ca4c570f1a","failure_pattern":"failed to cleanup resource group: failed waiting for resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: failed waiting for resourcegroup=\"autoscaling-cluster-2bvkgc\" to finish deleting: context canceled"}},{"row_id":"b1ecf6abad811364fb6d6ec7793cd5e3e554607fce7ba616a868552056479d9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507858162651136","occurred_at":"2026-03-16T11:37:08Z","failed_at":"e2e","signature_id":"0929f1268cda06834bce91a7b0b1e57a1f9b9b16300bacc6d9dd00df2a24e985","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a4a880\u003e: \n failed to create HCP cluster cluster-zstream-4-20-cdpvqt: failed waiting for cluster=\"cluster-zstream-4-20-cdpvqt\" in resourcegroup=\"rg-zstream-upgrade-4-20-bccpt6\" to finish creating: GET https://rp.j2651136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c51f19aa-b670-4113-9baa-6af8f159532b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c51f19aa-b670-4113-9baa-6af8f159532b\",\n \"name\": \"c51f19aa-b670-4113-9baa-6af8f159532b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:49:02.264347005Z\",\n \"endTime\": \"2026-03-16T12:50:00.815626772Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-138/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-138/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-20-cdpvqt: failed waiting for cluster=\\\"cluster-zstream-4-20-cdpvqt\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-bccpt6\\\" to finish creating: GET https://rp.j2651136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c51f19aa-b670-4113-9baa-6af8f159532b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c51f19aa-b670-4113-9baa-6af8f159532b\\\",\\n \\\"name\\\": \\\"c51f19aa-b670-4113-9baa-6af8f159532b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:49:02.264347005Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:50:00.815626772Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-138/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-138/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a4a860\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-cdpvqt\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-bccpt6\\\" to finish creating: GET https://rp.j2651136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c51f19aa-b670-4113-9baa-6af8f159532b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c51f19aa-b670-4113-9baa-6af8f159532b\\\",\\n \\\"name\\\": \\\"c51f19aa-b670-4113-9baa-6af8f159532b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:49:02.264347005Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:50:00.815626772Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-138/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-138/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: Subsc...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a4a880\u003e: \n failed to create HCP cluster cluster-zstream-4-20-cdpvqt: failed waiting for cluster=\"cluster-zstream-4-20-cdpvqt\" in resourcegroup=\"rg-zstream-upgrade-4-20-bccpt6\" to finish creating: GET https://rp.j2651136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c51f19aa-b670-4113-9baa-6af8f159532b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c51f19aa-b670-4113-9baa-6af8f159532b\",\n \"name\": \"c51f19aa-b670-4113-9baa-6af8f159532b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:49:02.264347005Z\",\n \"endTime\": \"2026-03-16T12:50:00.815626772Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-138/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-138/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-20-cdpvqt: failed waiting for cluster=\\\"cluster-zstream-4-20-cdpvqt\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-bccpt6\\\" to finish creating: GET https://rp.j2651136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c51f19aa-b670-4113-9baa-6af8f159532b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c51f19aa-b670-4113-9baa-6af8f159532b\\\",\\n \\\"name\\\": \\\"c51f19aa-b670-4113-9baa-6af8f159532b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:49:02.264347005Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:50:00.815626772Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-138/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-138/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a4a860\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-cdpvqt\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-bccpt6\\\" to finish creating: GET https://rp.j2651136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c51f19aa-b670-4113-9baa-6af8f159532b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c51f19aa-b670-4113-9baa-6af8f159532b\\\",\\n \\\"name\\\": \\\"c51f19aa-b670-4113-9baa-6af8f159532b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:49:02.264347005Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:50:00.815626772Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-138/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-138/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: Subsc...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bc388a1868b39499bfc4cb8d5e99e3b16772f8630c91ff6d3b44cec21ff0c10d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507858162651136","occurred_at":"2026-03-16T11:37:08Z","failed_at":"e2e","signature_id":"7944705127bd431eef77015fb8964580eadd1e3be88c9649b57bdc4e74d10ca9","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010b5dd0\u003e: \n failed to cleanup resource group: failed waiting for resourcegroup=\"pullsecret-test-bhtgnp\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000693200\u003e{\n msg: \"failed to cleanup resource group: failed waiting for resourcegroup=\\\"pullsecret-test-bhtgnp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006931c0\u003e{\n msg: \"failed waiting for resourcegroup=\\\"pullsecret-test-bhtgnp\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010b5dd0\u003e: \n failed to cleanup resource group: failed waiting for resourcegroup=\"pullsecret-test-bhtgnp\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000693200\u003e{\n msg: \"failed to cleanup resource group: failed waiting for resourcegroup=\\\"pullsecret-test-bhtgnp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006931c0\u003e{\n msg: \"failed waiting for resourcegroup=\\\"pullsecret-test-bhtgnp\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d55c4a29bbbf19a43335bfa2157018ebc82a1c8fe8cbea22f78ef2ca4c570f1a","failure_pattern":"failed to cleanup resource group: failed waiting for resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: failed waiting for resourcegroup=\"autoscaling-cluster-2bvkgc\" to finish deleting: context canceled"}},{"row_id":"e8f2a6c9ceb3f1b8ffdd69162048a2a961cabbdb704b246f2df500596b7127df","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507858162651136","occurred_at":"2026-03-16T11:37:08Z","failed_at":"e2e","signature_id":"8cb99bd4583c0d72930c3eb259db56bae44b5bd9c6dc2135d1634cf30a248f9a","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f36690\u003e: \n failed to cleanup resource group: failed waiting for resourcegroup=\"autoscaling-cluster-2bvkgc\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000857c60\u003e{\n msg: \"failed to cleanup resource group: failed waiting for resourcegroup=\\\"autoscaling-cluster-2bvkgc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000857c40\u003e{\n msg: \"failed waiting for resourcegroup=\\\"autoscaling-cluster-2bvkgc\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f36690\u003e: \n failed to cleanup resource group: failed waiting for resourcegroup=\"autoscaling-cluster-2bvkgc\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000857c60\u003e{\n msg: \"failed to cleanup resource group: failed waiting for resourcegroup=\\\"autoscaling-cluster-2bvkgc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000857c40\u003e{\n msg: \"failed waiting for resourcegroup=\\\"autoscaling-cluster-2bvkgc\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d55c4a29bbbf19a43335bfa2157018ebc82a1c8fe8cbea22f78ef2ca4c570f1a","failure_pattern":"failed to cleanup resource group: failed waiting for resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: failed waiting for resourcegroup=\"autoscaling-cluster-2bvkgc\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":15,"failure_pattern_ids":["503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","d55c4a29bbbf19a43335bfa2157018ebc82a1c8fe8cbea22f78ef2ca4c570f1a","d68a3a4000afced929cc779e7a36af528e5b7a9739bb3b6369253f0a6d739eb2"],"matched_occurrences":15,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507826227220480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4470,"pr_state":"open","pr_sha":"dfb641240c1f108be63315200630f51de45ba192","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:37:01Z"},"failed_at":["e2e"],"failed_test_count":18,"occurrences":[{"row_id":"016ef335ec15544fd56220fd3a2031dad98fa971dbae384e0359c4018cb9a411","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507826227220480","occurred_at":"2026-03-16T11:37:01Z","failed_at":"e2e","signature_id":"121a71112ec3544c96171322f88656121ff51845a7be22ebcd413d73a9e9a4fd","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e88380\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-mrbvrt\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/625b7db7-c87b-4386-b2f9-bc83bcf11a03\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/625b7db7-c87b-4386-b2f9-bc83bcf11a03\",\n \"name\": \"625b7db7-c87b-4386-b2f9-bc83bcf11a03\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:40:58.976812987Z\",\n \"endTime\": \"2026-03-16T12:41:58.144975207Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-222/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-222/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-mrbvrt\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/625b7db7-c87b-4386-b2f9-bc83bcf11a03\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/625b7db7-c87b-4386-b2f9-bc83bcf11a03\\\",\\n \\\"name\\\": \\\"625b7db7-c87b-4386-b2f9-bc83bcf11a03\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:40:58.976812987Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:41:58.144975207Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-222/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-222/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e88280\u003e{\n msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-mrbvrt\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/625b7db7-c87b-4386-b2f9-bc83bcf11a03\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/625b7db7-c87b-4386-b2f9-bc83bcf11a03\\\",\\n \\\"name\\\": \\\"625b7db7-c87b-4386-b2f9-bc83bcf11a03\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:40:58.976812987Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:41:58.144975207Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-222/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-222/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"e...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e88380\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-mrbvrt\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/625b7db7-c87b-4386-b2f9-bc83bcf11a03\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/625b7db7-c87b-4386-b2f9-bc83bcf11a03\",\n \"name\": \"625b7db7-c87b-4386-b2f9-bc83bcf11a03\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:40:58.976812987Z\",\n \"endTime\": \"2026-03-16T12:41:58.144975207Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-222/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-222/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-mrbvrt\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/625b7db7-c87b-4386-b2f9-bc83bcf11a03\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/625b7db7-c87b-4386-b2f9-bc83bcf11a03\\\",\\n \\\"name\\\": \\\"625b7db7-c87b-4386-b2f9-bc83bcf11a03\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:40:58.976812987Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:41:58.144975207Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-222/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-222/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e88280\u003e{\n msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-mrbvrt\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/625b7db7-c87b-4386-b2f9-bc83bcf11a03\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/625b7db7-c87b-4386-b2f9-bc83bcf11a03\\\",\\n \\\"name\\\": \\\"625b7db7-c87b-4386-b2f9-bc83bcf11a03\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:40:58.976812987Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:41:58.144975207Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-222/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-222/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"e...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"01dee07a0ea6e052fddcae79793a9af84dd28141346ab99a6ed09a997238fa64","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507826227220480","occurred_at":"2026-03-16T11:37:01Z","failed_at":"e2e","signature_id":"98feb7431eee3576bf5e60e89726b5e349809e09f0b398b2c50cd316eb802fbe","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000d870e0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-tbnv77\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa903328-1672-4406-a5b2-4a1df3f220de\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa903328-1672-4406-a5b2-4a1df3f220de\",\n \"name\": \"aa903328-1672-4406-a5b2-4a1df3f220de\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:45:59.548150295Z\",\n \"endTime\": \"2026-03-16T12:48:12.416287908Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-283/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-283/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '2' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-tbnv77\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa903328-1672-4406-a5b2-4a1df3f220de\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa903328-1672-4406-a5b2-4a1df3f220de\\\",\\n \\\"name\\\": \\\"aa903328-1672-4406-a5b2-4a1df3f220de\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:45:59.548150295Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:48:12.416287908Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-283/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-283/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '2' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000d870c0\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-tbnv77\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa903328-1672-4406-a5b2-4a1df3f220de\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa903328-1672-4406-a5b2-4a1df3f220de\\\",\\n \\\"name\\\": \\\"aa903328-1672-4406-a5b2-4a1df3f220de\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:45:59.548150295Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:48:12.416287908Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-283/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-283/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-------------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000d870e0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-tbnv77\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa903328-1672-4406-a5b2-4a1df3f220de\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa903328-1672-4406-a5b2-4a1df3f220de\",\n \"name\": \"aa903328-1672-4406-a5b2-4a1df3f220de\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:45:59.548150295Z\",\n \"endTime\": \"2026-03-16T12:48:12.416287908Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-283/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-283/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '2' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-tbnv77\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa903328-1672-4406-a5b2-4a1df3f220de\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa903328-1672-4406-a5b2-4a1df3f220de\\\",\\n \\\"name\\\": \\\"aa903328-1672-4406-a5b2-4a1df3f220de\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:45:59.548150295Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:48:12.416287908Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-283/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-283/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '2' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000d870c0\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-tbnv77\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa903328-1672-4406-a5b2-4a1df3f220de\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa903328-1672-4406-a5b2-4a1df3f220de\\\",\\n \\\"name\\\": \\\"aa903328-1672-4406-a5b2-4a1df3f220de\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:45:59.548150295Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:48:12.416287908Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-283/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-283/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-------------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"066f69aec09b2bedd1911fa4864335570d727bd64ed83686e1b6b8fa8c342749","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507826227220480","occurred_at":"2026-03-16T11:37:01Z","failed_at":"e2e","signature_id":"d1d72e65e602c9f0658784eaa76023649ef69f7491eb716cd2e67c6eb8e4296b","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00136c380\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-9jsvk2\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/120e5a0f-75bd-4759-8ece-4c9c94be6614\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/120e5a0f-75bd-4759-8ece-4c9c94be6614\",\n \"name\": \"120e5a0f-75bd-4759-8ece-4c9c94be6614\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:40:57.249642697Z\",\n \"endTime\": \"2026-03-16T12:41:58.288273824Z\",\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"Cluster provisioning failed due to multiple errors\",\n \"details\": [\n {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-197/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-197/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n },\n {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'arm64-vm-customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-9jsvk2/providers/Microsoft.Network/networkSecurityGroups/arm64-vm-customer-nsg\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-9jsvk2\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/120e5a0f-75bd-4759-8ece-4c9c94be6614\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/120e5a0f-75bd-4759-8ece-4c9c94be6614\\\",\\n \\\"name\\\": \\\"120e5a0f-75bd-4759-8ece-4c9c94be6614\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:40:57.249642697Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:41:58.288273824Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-197/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-197/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n },\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'arm64-vm-customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-9jsvk2/providers/Microsoft.Network/networkSecurityGroups/arm64-vm-customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00136c360\u003e{\n msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-9jsvk2\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/120e5a0f-75bd-4759-8ece-4c9c94be6614\\n---------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00136c380\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-9jsvk2\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/120e5a0f-75bd-4759-8ece-4c9c94be6614\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/120e5a0f-75bd-4759-8ece-4c9c94be6614\",\n \"name\": \"120e5a0f-75bd-4759-8ece-4c9c94be6614\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:40:57.249642697Z\",\n \"endTime\": \"2026-03-16T12:41:58.288273824Z\",\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"Cluster provisioning failed due to multiple errors\",\n \"details\": [\n {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-197/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-197/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n },\n {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'arm64-vm-customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-9jsvk2/providers/Microsoft.Network/networkSecurityGroups/arm64-vm-customer-nsg\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-9jsvk2\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/120e5a0f-75bd-4759-8ece-4c9c94be6614\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/120e5a0f-75bd-4759-8ece-4c9c94be6614\\\",\\n \\\"name\\\": \\\"120e5a0f-75bd-4759-8ece-4c9c94be6614\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:40:57.249642697Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:41:58.288273824Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-197/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-197/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n },\\n {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'arm64-vm-customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-9jsvk2/providers/Microsoft.Network/networkSecurityGroups/arm64-vm-customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00136c360\u003e{\n msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-9jsvk2\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/120e5a0f-75bd-4759-8ece-4c9c94be6614\\n---------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0aa9ed5d94e7167a12aa30282eff19539fc3fc473772cb96b223d420df66775f","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"22273d600ca9575097aaff9fa57b3b7ffe85506fb9262ce2251f341891add341","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507826227220480","occurred_at":"2026-03-16T11:37:01Z","failed_at":"e2e","signature_id":"a68e944e1237f1e0e5f76c730be2069c06ae43e1909e4a04c4f2202d2762dca5","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ede8d0\u003e: \n failed to cleanup resource group: failed waiting for resourcegroup=\"autoscaling-cluster-9x6bsc\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001399540\u003e{\n msg: \"failed to cleanup resource group: failed waiting for resourcegroup=\\\"autoscaling-cluster-9x6bsc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001399520\u003e{\n msg: \"failed waiting for resourcegroup=\\\"autoscaling-cluster-9x6bsc\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ede8d0\u003e: \n failed to cleanup resource group: failed waiting for resourcegroup=\"autoscaling-cluster-9x6bsc\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001399540\u003e{\n msg: \"failed to cleanup resource group: failed waiting for resourcegroup=\\\"autoscaling-cluster-9x6bsc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001399520\u003e{\n msg: \"failed waiting for resourcegroup=\\\"autoscaling-cluster-9x6bsc\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d55c4a29bbbf19a43335bfa2157018ebc82a1c8fe8cbea22f78ef2ca4c570f1a","failure_pattern":"failed to cleanup resource group: failed waiting for resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: failed waiting for resourcegroup=\"autoscaling-cluster-2bvkgc\" to finish deleting: context canceled"}},{"row_id":"2c7ba97899b3264454fc527ed450d948f9941bdd44fb3438e2ad688ba766dbe8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507826227220480","occurred_at":"2026-03-16T11:37:01Z","failed_at":"e2e","signature_id":"d51695aaf667d0661952f0b54faef413d8a97659007b65c73d3d6e5b5bf49b38","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00111a150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-xw5rxt\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000176640\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-xw5rxt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000176600\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-xw5rxt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0001765c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-xw5rxt\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00111a150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-xw5rxt\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000176640\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-xw5rxt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000176600\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-xw5rxt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0001765c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-xw5rxt\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"2dbd2356a3bd49162486614f005f8943af7136c14b6a3aebebe799e0aacebc8f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507826227220480","occurred_at":"2026-03-16T11:37:01Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:152]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:152]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"40aed61881c7703e1a121348b4e5c7d26e44ca366504c202b0d66b24f33fe80a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507826227220480","occurred_at":"2026-03-16T11:37:01Z","failed_at":"e2e","signature_id":"0097bcd0464d79d249de4dadb358a2200a988bf8a75ed3575f40c86744746c48","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0012de1b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-j4dj27\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000ddd9c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-j4dj27\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ddd980\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-j4dj27\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ddd940\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-j4dj27\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0012de1b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-j4dj27\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000ddd9c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-j4dj27\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ddd980\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-j4dj27\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ddd940\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-j4dj27\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"474c0b4ecb196c7190d40ce2f0930bd222d946e82e10fbd557bf9281eb9544bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507826227220480","occurred_at":"2026-03-16T11:37:01Z","failed_at":"e2e","signature_id":"208c1d000c1f416860ef6e438bbbb8cd78075f14813e209d7a81787dbfdab7a2","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000c4c360\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-r2jsvz\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/577c0a8d-dcc6-4d43-97f0-70c20b633e7d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/577c0a8d-dcc6-4d43-97f0-70c20b633e7d\",\n \"name\": \"577c0a8d-dcc6-4d43-97f0-70c20b633e7d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:41:08.167401696Z\",\n \"endTime\": \"2026-03-16T12:41:58.502094725Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-263/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-263/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster ea-cluster: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-r2jsvz\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/577c0a8d-dcc6-4d43-97f0-70c20b633e7d\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/577c0a8d-dcc6-4d43-97f0-70c20b633e7d\\\",\\n \\\"name\\\": \\\"577c0a8d-dcc6-4d43-97f0-70c20b633e7d\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:41:08.167401696Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:41:58.502094725Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-263/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-263/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000c4c340\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-r2jsvz\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/577c0a8d-dcc6-4d43-97f0-70c20b633e7d\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/577c0a8d-dcc6-4d43-97f0-70c20b633e7d\\\",\\n \\\"name\\\": \\\"577c0a8d-dcc6-4d43-97f0-70c20b633e7d\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:41:08.167401696Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:41:58.502094725Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-263/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-263/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000c4c360\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-r2jsvz\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/577c0a8d-dcc6-4d43-97f0-70c20b633e7d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/577c0a8d-dcc6-4d43-97f0-70c20b633e7d\",\n \"name\": \"577c0a8d-dcc6-4d43-97f0-70c20b633e7d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:41:08.167401696Z\",\n \"endTime\": \"2026-03-16T12:41:58.502094725Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-263/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-263/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster ea-cluster: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-r2jsvz\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/577c0a8d-dcc6-4d43-97f0-70c20b633e7d\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/577c0a8d-dcc6-4d43-97f0-70c20b633e7d\\\",\\n \\\"name\\\": \\\"577c0a8d-dcc6-4d43-97f0-70c20b633e7d\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:41:08.167401696Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:41:58.502094725Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-263/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-263/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000c4c340\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-r2jsvz\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/577c0a8d-dcc6-4d43-97f0-70c20b633e7d\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/577c0a8d-dcc6-4d43-97f0-70c20b633e7d\\\",\\n \\\"name\\\": \\\"577c0a8d-dcc6-4d43-97f0-70c20b633e7d\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:41:08.167401696Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:41:58.502094725Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-263/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-263/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"702f38035b08d416159de249391cf9476d4155bca512afe1fc61721d08731622","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507826227220480","occurred_at":"2026-03-16T11:37:01Z","failed_at":"e2e","signature_id":"c43eaf02e9f03a6c59f5f7040f04111130c84ce5da6fb03bf163133dbd7ae3e3","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129]: Unexpected error:\n \u003c*exported.ResponseError | 0xc0004cfe00\u003e: \n GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2bf8b42e-2ebc-476b-b395-b387e0daa355\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2bf8b42e-2ebc-476b-b395-b387e0daa355\",\n \"name\": \"2bf8b42e-2ebc-476b-b395-b387e0daa355\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:41:56.025534294Z\",\n \"endTime\": \"2026-03-16T12:42:51.560044029Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-251/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-251/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"997db521-3d5d-46d8-b08f-8b1664aa2128\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 12:42:56 GMT\",\n ],\n \"Content-Length\": [\"1565\"],\n \"X-Envoy-Upstream-Service-Time\": [\"8\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000a91140\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 50, 98, 102, 56, 98, 52, 50, 101, 45, 50, 101, 98, 99, 45, 52, 55, 54, 98, 45, 98, 51, 57, 53, 45, 98, 51, 56, 55, 101, 48, 100, 97, 97, 51, 53, 53, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 50, 98, 102, 56, 98, 52, 50, 101, 45, 50, 101, 98, 99, 45, 52, 55, 54, 98, 45, 98, 51, 57, 53, 45, 98, 51, 56, 55, 101, 48, 100, 97, 97, 51, 53, 53, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 116, 117, 115, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 114, 116, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 54, 84, 49, 50, 58, 52, 49, 58, 53, 54, 46, 48, 50, 53, 53, 51, 52, 50, 57, 52, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 110, 100, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 54, 84, 49, 50, 58, 52, 50, 58, 53, 49, 46, 53, 54, 48, 48, 52, 52, 48, 50, 57, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 102, 97, 105, 108, 101, 100, 32, 116, 111, 32, 103, 101, 116, 32, 109, 97, 110, 97, 103, 101, 100, 32, 105, 100, 101, 110, 116, 105, 116, 121, 32, 39, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 97, 114, 111, 45, 104, 99, 112, 45, 116, 101, 115, 116, 45, 109, 115, 105, 45, 99, 111, 110, 116, 97, 105, 110, 101, 114, 115, 45, 100, 101, 118, 45, 50, 53, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 77, 97, 110, 97, 103, 101, 100, 73, 100, 101, 110, 116, 105, 116, 121, 47, 117, 115, 101, 114, 65, 115, 115, 105, 103, 110, 101, 100, 73, 100, 101, 110, 116, 105, 116, 105, 101, 115, 47, 100, 105, 115, 107, 45, 99, 115, 105, 45, 100, 114, 105, 118, 101, 114, 39, 58, 32, 71, 69, 84, 32, 104, 116, 116, 112, 115, 58, 47, 47, 109, 97, 110, 97, 103, 101, 109, 101, 110, 116, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129]: Unexpected error:\n \u003c*exported.ResponseError | 0xc0004cfe00\u003e: \n GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2bf8b42e-2ebc-476b-b395-b387e0daa355\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2bf8b42e-2ebc-476b-b395-b387e0daa355\",\n \"name\": \"2bf8b42e-2ebc-476b-b395-b387e0daa355\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:41:56.025534294Z\",\n \"endTime\": \"2026-03-16T12:42:51.560044029Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-251/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-251/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"997db521-3d5d-46d8-b08f-8b1664aa2128\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 12:42:56 GMT\",\n ],\n \"Content-Length\": [\"1565\"],\n \"X-Envoy-Upstream-Service-Time\": [\"8\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000a91140\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 50, 98, 102, 56, 98, 52, 50, 101, 45, 50, 101, 98, 99, 45, 52, 55, 54, 98, 45, 98, 51, 57, 53, 45, 98, 51, 56, 55, 101, 48, 100, 97, 97, 51, 53, 53, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 50, 98, 102, 56, 98, 52, 50, 101, 45, 50, 101, 98, 99, 45, 52, 55, 54, 98, 45, 98, 51, 57, 53, 45, 98, 51, 56, 55, 101, 48, 100, 97, 97, 51, 53, 53, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 116, 117, 115, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 114, 116, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 54, 84, 49, 50, 58, 52, 49, 58, 53, 54, 46, 48, 50, 53, 53, 51, 52, 50, 57, 52, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 110, 100, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 54, 84, 49, 50, 58, 52, 50, 58, 53, 49, 46, 53, 54, 48, 48, 52, 52, 48, 50, 57, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 102, 97, 105, 108, 101, 100, 32, 116, 111, 32, 103, 101, 116, 32, 109, 97, 110, 97, 103, 101, 100, 32, 105, 100, 101, 110, 116, 105, 116, 121, 32, 39, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 97, 114, 111, 45, 104, 99, 112, 45, 116, 101, 115, 116, 45, 109, 115, 105, 45, 99, 111, 110, 116, 97, 105, 110, 101, 114, 115, 45, 100, 101, 118, 45, 50, 53, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 77, 97, 110, 97, 103, 101, 100, 73, 100, 101, 110, 116, 105, 116, 121, 47, 117, 115, 101, 114, 65, 115, 115, 105, 103, 110, 101, 100, 73, 100, 101, 110, 116, 105, 116, 105, 101, 115, 47, 100, 105, 115, 107, 45, 99, 115, 105, 45, 100, 114, 105, 118, 101, 114, 39, 58, 32, 71, 69, 84, 32, 104, 116, 116, 112, 115, 58, 47, 47, 109, 97, 110, 97, 103, 101, 109, 101, 110, 116, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"825ee288ac57f20771f9c9cfc71e97ecad323721dfbea62486b1a595e67f98c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507826227220480","occurred_at":"2026-03-16T11:37:01Z","failed_at":"e2e","signature_id":"73634a82ba869bcb7466c6dd5b5253ee867492d7f62a98549cae2d597a3d2ade","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.009s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001751a60\u003e: \n missing expected log sources: [ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [ocm-arohcp/kube-apiserver]\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.009s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001751a60\u003e: \n missing expected log sources: [ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [ocm-arohcp/kube-apiserver]\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d68a3a4000afced929cc779e7a36af528e5b7a9739bb3b6369253f0a6d739eb2","failure_pattern":"missing expected log sources: [ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [ocm-arohcp/kube-apiserver]"}},{"row_id":"89efa85344169f432fa79ebddb0c91b7253b654e9c76d4a4754f6d7ba0fd3134","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507826227220480","occurred_at":"2026-03-16T11:37:01Z","failed_at":"e2e","signature_id":"bb7fafce5eeb4386fc17618d7d53cafad5da7a7e8f16ae785170e7609972a34d","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d89cb0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-19-7nzxrw\" in resourcegroup=\"rg-zstream-upgrade-4-19-pfzszd\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a580a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-19-7nzxrw\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-pfzszd\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a58080\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-19-7nzxrw\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-pfzszd\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a58060\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-7nzxrw\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-pfzszd\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d89cb0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-19-7nzxrw\" in resourcegroup=\"rg-zstream-upgrade-4-19-pfzszd\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a580a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-19-7nzxrw\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-pfzszd\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a58080\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-19-7nzxrw\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-pfzszd\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a58060\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-7nzxrw\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-pfzszd\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"93811a8edb0c04e76cc64c44c82163dd7d20efd925d97f3c50bfd61292c76238","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507826227220480","occurred_at":"2026-03-16T11:37:01Z","failed_at":"e2e","signature_id":"a33909f9e575438bdeafe2e79b705c3e9494978bb9cfeb75138781d75d20a98c","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f50e70\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-hjsdwz\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007e7ca0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-hjsdwz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007e7c80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-hjsdwz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007e7c60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-hjsdwz\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f50e70\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-hjsdwz\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007e7ca0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-hjsdwz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007e7c80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-hjsdwz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007e7c60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-hjsdwz\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"9543cbbc0e5a863e6fec2e1c388718f83761901e13a68380d97589b7688fb571","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507826227220480","occurred_at":"2026-03-16T11:37:01Z","failed_at":"e2e","signature_id":"0bf5af1070d9c22bab3561020d3addfb04892e54f4b3750fe3c480a800fd15b9","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ee81c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-t49wwc\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0004a6620\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-t49wwc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0004a6600\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-t49wwc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0004a6520\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-t49wwc\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ee81c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-t49wwc\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0004a6620\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-t49wwc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0004a6600\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-t49wwc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0004a6520\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-t49wwc\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"a1530d5b05c750df6338030601c21903ed71a9aac66e9b8ce0839fd1ee04d94f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507826227220480","occurred_at":"2026-03-16T11:37:01Z","failed_at":"e2e","signature_id":"0c49ca273112000d65ba2ed8e1e21123be25191bcc4579c06de82632ba7d4e80","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:89]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e84d60\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-zdjk7d\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74f07a94-3ad2-43f0-b896-ab9ac9199f16\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74f07a94-3ad2-43f0-b896-ab9ac9199f16\",\n \"name\": \"74f07a94-3ad2-43f0-b896-ab9ac9199f16\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:43:33.474385131Z\",\n \"endTime\": \"2026-03-16T12:44:39.436485773Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-159/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-159/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-zdjk7d\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74f07a94-3ad2-43f0-b896-ab9ac9199f16\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74f07a94-3ad2-43f0-b896-ab9ac9199f16\\\",\\n \\\"name\\\": \\\"74f07a94-3ad2-43f0-b896-ab9ac9199f16\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:43:33.474385131Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:44:39.436485773Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-159/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-159/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e84d40\u003e{\n msg: \"failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-zdjk7d\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74f07a94-3ad2-43f0-b896-ab9ac9199f16\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74f07a94-3ad2-43f0-b896-ab9ac9199f16\\\",\\n \\\"name\\\": \\\"74f07a94-3ad2-43f0-b896-ab9ac9199f16\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:43:33.474385131Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:44:39.436485773Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-159/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-159/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:89]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e84d60\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-zdjk7d\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74f07a94-3ad2-43f0-b896-ab9ac9199f16\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74f07a94-3ad2-43f0-b896-ab9ac9199f16\",\n \"name\": \"74f07a94-3ad2-43f0-b896-ab9ac9199f16\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:43:33.474385131Z\",\n \"endTime\": \"2026-03-16T12:44:39.436485773Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-159/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-159/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-zdjk7d\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74f07a94-3ad2-43f0-b896-ab9ac9199f16\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74f07a94-3ad2-43f0-b896-ab9ac9199f16\\\",\\n \\\"name\\\": \\\"74f07a94-3ad2-43f0-b896-ab9ac9199f16\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:43:33.474385131Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:44:39.436485773Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-159/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-159/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e84d40\u003e{\n msg: \"failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-zdjk7d\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74f07a94-3ad2-43f0-b896-ab9ac9199f16\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74f07a94-3ad2-43f0-b896-ab9ac9199f16\\\",\\n \\\"name\\\": \\\"74f07a94-3ad2-43f0-b896-ab9ac9199f16\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:43:33.474385131Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:44:39.436485773Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-159/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-159/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ae5deafd7d170fa0cdfc5c41e0e2a8d8b7142d0dd71714c6df23d7b2d007aba4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507826227220480","occurred_at":"2026-03-16T11:37:01Z","failed_at":"e2e","signature_id":"f2b9961ff238e6e40aaee006dad327244ae627f9a6316b4235fd2db16d8eb42f","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000f85740\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-pxgn8m\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67188cb4-37b8-4ab8-b4f7-1402512d327a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67188cb4-37b8-4ab8-b4f7-1402512d327a\",\n \"name\": \"67188cb4-37b8-4ab8-b4f7-1402512d327a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:40:59.142803014Z\",\n \"endTime\": \"2026-03-16T12:41:58.467762398Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-273/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-273/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-pxgn8m\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67188cb4-37b8-4ab8-b4f7-1402512d327a\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67188cb4-37b8-4ab8-b4f7-1402512d327a\\\",\\n \\\"name\\\": \\\"67188cb4-37b8-4ab8-b4f7-1402512d327a\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:40:59.142803014Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:41:58.467762398Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-273/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-273/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000f85720\u003e{\n msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-pxgn8m\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67188cb4-37b8-4ab8-b4f7-1402512d327a\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67188cb4-37b8-4ab8-b4f7-1402512d327a\\\",\\n \\\"name\\\": \\\"67188cb4-37b8-4ab8-b4f7-1402512d327a\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:40:59.142803014Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:41:58.467762398Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-273/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-273/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n----...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000f85740\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-pxgn8m\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67188cb4-37b8-4ab8-b4f7-1402512d327a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67188cb4-37b8-4ab8-b4f7-1402512d327a\",\n \"name\": \"67188cb4-37b8-4ab8-b4f7-1402512d327a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:40:59.142803014Z\",\n \"endTime\": \"2026-03-16T12:41:58.467762398Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-273/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-273/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-pxgn8m\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67188cb4-37b8-4ab8-b4f7-1402512d327a\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67188cb4-37b8-4ab8-b4f7-1402512d327a\\\",\\n \\\"name\\\": \\\"67188cb4-37b8-4ab8-b4f7-1402512d327a\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:40:59.142803014Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:41:58.467762398Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-273/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-273/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000f85720\u003e{\n msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-pxgn8m\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67188cb4-37b8-4ab8-b4f7-1402512d327a\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67188cb4-37b8-4ab8-b4f7-1402512d327a\\\",\\n \\\"name\\\": \\\"67188cb4-37b8-4ab8-b4f7-1402512d327a\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:40:59.142803014Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:41:58.467762398Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-273/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-273/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n----...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b7423ad46f80df6dd348da1c40fe65ea786e46d35f8917a8bbb00fb482b5d0ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507826227220480","occurred_at":"2026-03-16T11:37:01Z","failed_at":"e2e","signature_id":"b36384b67b62029b1206c7937dd6cc7133d16cf3b7d27f500d041e32618332c8","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000db0440\u003e: \n failed to create HCP cluster cluster-zstream-4-21-tjnwkc: failed waiting for cluster=\"cluster-zstream-4-21-tjnwkc\" in resourcegroup=\"rg-zstream-upgrade-4-21-xgkq8d\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/94d48c6b-47fa-4a94-9a31-f62dfadfe9a9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/94d48c6b-47fa-4a94-9a31-f62dfadfe9a9\",\n \"name\": \"94d48c6b-47fa-4a94-9a31-f62dfadfe9a9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:46:09.542833596Z\",\n \"endTime\": \"2026-03-16T12:47:06.512450556Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-197/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-197/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-21-tjnwkc: failed waiting for cluster=\\\"cluster-zstream-4-21-tjnwkc\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-xgkq8d\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/94d48c6b-47fa-4a94-9a31-f62dfadfe9a9\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/94d48c6b-47fa-4a94-9a31-f62dfadfe9a9\\\",\\n \\\"name\\\": \\\"94d48c6b-47fa-4a94-9a31-f62dfadfe9a9\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:46:09.542833596Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:47:06.512450556Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-197/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-197/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000db0420\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-tjnwkc\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-xgkq8d\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/94d48c6b-47fa-4a94-9a31-f62dfadfe9a9\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/94d48c6b-47fa-4a94-9a31-f62dfadfe9a9\\\",\\n \\\"name\\\": \\\"94d48c6b-47fa-4a94-9a31-f62dfadfe9a9\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:46:09.542833596Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:47:06.512450556Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-197/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-197/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000db0440\u003e: \n failed to create HCP cluster cluster-zstream-4-21-tjnwkc: failed waiting for cluster=\"cluster-zstream-4-21-tjnwkc\" in resourcegroup=\"rg-zstream-upgrade-4-21-xgkq8d\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/94d48c6b-47fa-4a94-9a31-f62dfadfe9a9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/94d48c6b-47fa-4a94-9a31-f62dfadfe9a9\",\n \"name\": \"94d48c6b-47fa-4a94-9a31-f62dfadfe9a9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:46:09.542833596Z\",\n \"endTime\": \"2026-03-16T12:47:06.512450556Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-197/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-197/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-21-tjnwkc: failed waiting for cluster=\\\"cluster-zstream-4-21-tjnwkc\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-xgkq8d\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/94d48c6b-47fa-4a94-9a31-f62dfadfe9a9\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/94d48c6b-47fa-4a94-9a31-f62dfadfe9a9\\\",\\n \\\"name\\\": \\\"94d48c6b-47fa-4a94-9a31-f62dfadfe9a9\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:46:09.542833596Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:47:06.512450556Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-197/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-197/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000db0420\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-tjnwkc\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-xgkq8d\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/94d48c6b-47fa-4a94-9a31-f62dfadfe9a9\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/94d48c6b-47fa-4a94-9a31-f62dfadfe9a9\\\",\\n \\\"name\\\": \\\"94d48c6b-47fa-4a94-9a31-f62dfadfe9a9\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:46:09.542833596Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:47:06.512450556Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-197/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-197/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b7a832890900de0e2ae291a08003ba4345766d22e07f58be1b4fe8d817e48514","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507826227220480","occurred_at":"2026-03-16T11:37:01Z","failed_at":"e2e","signature_id":"7ee66183f9f4151b631c1d77d80b499d6ac44273d5bf94ed67284ee318e598f7","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0008c4680\u003e: \n failed to create HCP cluster ea-listm2rzdz: failed waiting for cluster=\"ea-listm2rzdz\" in resourcegroup=\"ea-list-2tsj85\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da6244ff-1caa-4262-a5b2-f747dd758fcf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da6244ff-1caa-4262-a5b2-f747dd758fcf\",\n \"name\": \"da6244ff-1caa-4262-a5b2-f747dd758fcf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:40:57.66091959Z\",\n \"endTime\": \"2026-03-16T12:41:58.815134562Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-283/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-283/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster ea-listm2rzdz: failed waiting for cluster=\\\"ea-listm2rzdz\\\" in resourcegroup=\\\"ea-list-2tsj85\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da6244ff-1caa-4262-a5b2-f747dd758fcf\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da6244ff-1caa-4262-a5b2-f747dd758fcf\\\",\\n \\\"name\\\": \\\"da6244ff-1caa-4262-a5b2-f747dd758fcf\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:40:57.66091959Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:41:58.815134562Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-283/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-283/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0008c4660\u003e{\n msg: \"failed waiting for cluster=\\\"ea-listm2rzdz\\\" in resourcegroup=\\\"ea-list-2tsj85\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da6244ff-1caa-4262-a5b2-f747dd758fcf\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da6244ff-1caa-4262-a5b2-f747dd758fcf\\\",\\n \\\"name\\\": \\\"da6244ff-1caa-4262-a5b2-f747dd758fcf\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:40:57.66091959Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:41:58.815134562Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-283/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-283/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0008c4680\u003e: \n failed to create HCP cluster ea-listm2rzdz: failed waiting for cluster=\"ea-listm2rzdz\" in resourcegroup=\"ea-list-2tsj85\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da6244ff-1caa-4262-a5b2-f747dd758fcf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da6244ff-1caa-4262-a5b2-f747dd758fcf\",\n \"name\": \"da6244ff-1caa-4262-a5b2-f747dd758fcf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:40:57.66091959Z\",\n \"endTime\": \"2026-03-16T12:41:58.815134562Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-283/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-283/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster ea-listm2rzdz: failed waiting for cluster=\\\"ea-listm2rzdz\\\" in resourcegroup=\\\"ea-list-2tsj85\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da6244ff-1caa-4262-a5b2-f747dd758fcf\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da6244ff-1caa-4262-a5b2-f747dd758fcf\\\",\\n \\\"name\\\": \\\"da6244ff-1caa-4262-a5b2-f747dd758fcf\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:40:57.66091959Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:41:58.815134562Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-283/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-283/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0008c4660\u003e{\n msg: \"failed waiting for cluster=\\\"ea-listm2rzdz\\\" in resourcegroup=\\\"ea-list-2tsj85\\\" to finish creating: GET https://rp.j7220480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da6244ff-1caa-4262-a5b2-f747dd758fcf\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da6244ff-1caa-4262-a5b2-f747dd758fcf\\\",\\n \\\"name\\\": \\\"da6244ff-1caa-4262-a5b2-f747dd758fcf\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:40:57.66091959Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:41:58.815134562Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-283/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-283/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bea6a2506d3365c261b313cf924a776d954019184e40cbc9f5b18f169a3b5ba9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507826227220480","occurred_at":"2026-03-16T11:37:01Z","failed_at":"e2e","signature_id":"05df15b4963428235afea44cb668d2d36efbeb6a3a899b03a5059263f9bff22b","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:390]: Timed out after 600.000s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:389 with:\nAll ClusterOperators should report Available=True, but these are not available: [image-registry (Available=False)]\nExpected\n \u003c[]string | len:1, cap:1\u003e: [\n \"image-registry (Available=False)\",\n ]\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:390]: Timed out after 600.000s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:389 with:\nAll ClusterOperators should report Available=True, but these are not available: [image-registry (Available=False)]\nExpected\n \u003c[]string | len:1, cap:1\u003e: [\n \"image-registry (Available=False)\",\n ]\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d24e138f3b84b77b0ea6793407ac1f41343d0658f0c69312eac159e4f4f7eb45","failure_pattern":"All ClusterOperators should report Available=True, but these are not available: [image-registry (Available=False)]","search_query":"All ClusterOperators should report Available=True, but these are not available: [image-registry (Available=False)]"}}],"failure_pattern_summary":{"signature_count":18,"failure_pattern_ids":["0aa9ed5d94e7167a12aa30282eff19539fc3fc473772cb96b223d420df66775f","503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","d24e138f3b84b77b0ea6793407ac1f41343d0658f0c69312eac159e4f4f7eb45","d55c4a29bbbf19a43335bfa2157018ebc82a1c8fe8cbea22f78ef2ca4c570f1a","d68a3a4000afced929cc779e7a36af528e5b7a9739bb3b6369253f0a6d739eb2"],"matched_occurrences":18,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507792735703040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4473,"pr_state":"closed","pr_sha":"bec5815ca7dc3703fc750e5f2ef845414e297409","final_merged_sha":"8b7576c81d01104dc238546ec2678952bf7afd61","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:36:53Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"0e082b929cb28f7bb73cff8f1348439f4b75f760468b822babecbf25c62f3c94","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507792735703040","occurred_at":"2026-03-16T11:36:53Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507788302323712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4477,"pr_state":"open","pr_sha":"79968855b403781872aa32d09e475edd3cbcd658","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:36:52Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"fd997ab57cd3b5f0d7ee92422d667ab8a86037c95f3b75a9580ea371156e403e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507788302323712","occurred_at":"2026-03-16T11:36:52Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4478/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507756547248128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4478,"pr_state":"closed","pr_sha":"e0b9c949795eb1f78de3754aed0733e48d81557c","final_merged_sha":"e0b9c949795eb1f78de3754aed0733e48d81557c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T11:36:44Z"},"failed_at":["e2e"],"failed_test_count":15,"occurrences":[{"row_id":"054b65f6a1aed5e450119ca1ee78451a7098b17fed491a4034b12da043defa91","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4478/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507756547248128","occurred_at":"2026-03-16T11:36:44Z","failed_at":"e2e","signature_id":"460e226777500d226b9eb35a2aff2159b8a950468786901bb31be58f8d64a774","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:96]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0005a0660\u003e: \n failed to create HCP cluster basic-hcp-cluster: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-pslrw6\" to finish creating: GET https://rp.j7248128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2cb08497-c1a3-40ae-8441-14884f9719af\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2cb08497-c1a3-40ae-8441-14884f9719af\",\n \"name\": \"2cb08497-c1a3-40ae-8441-14884f9719af\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:39:38.97159587Z\",\n \"endTime\": \"2026-03-16T12:41:03.165372775Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-215/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-215/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster basic-hcp-cluster: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-pslrw6\\\" to finish creating: GET https://rp.j7248128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2cb08497-c1a3-40ae-8441-14884f9719af\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2cb08497-c1a3-40ae-8441-14884f9719af\\\",\\n \\\"name\\\": \\\"2cb08497-c1a3-40ae-8441-14884f9719af\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:39:38.97159587Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:41:03.165372775Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-215/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-215/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0005a0640\u003e{\n msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-pslrw6\\\" to finish creating: GET https://rp.j7248128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2cb08497-c1a3-40ae-8441-14884f9719af\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2cb08497-c1a3-40ae-8441-14884f9719af\\\",\\n \\\"name\\\": \\\"2cb08497-c1a3-40ae-8441-14884f9719af\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:39:38.97159587Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:41:03.165372775Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-215/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-215/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-----------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:96]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0005a0660\u003e: \n failed to create HCP cluster basic-hcp-cluster: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-pslrw6\" to finish creating: GET https://rp.j7248128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2cb08497-c1a3-40ae-8441-14884f9719af\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2cb08497-c1a3-40ae-8441-14884f9719af\",\n \"name\": \"2cb08497-c1a3-40ae-8441-14884f9719af\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-16T12:39:38.97159587Z\",\n \"endTime\": \"2026-03-16T12:41:03.165372775Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-215/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-215/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster basic-hcp-cluster: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-pslrw6\\\" to finish creating: GET https://rp.j7248128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2cb08497-c1a3-40ae-8441-14884f9719af\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2cb08497-c1a3-40ae-8441-14884f9719af\\\",\\n \\\"name\\\": \\\"2cb08497-c1a3-40ae-8441-14884f9719af\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:39:38.97159587Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:41:03.165372775Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-215/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-215/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0005a0640\u003e{\n msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-pslrw6\\\" to finish creating: GET https://rp.j7248128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2cb08497-c1a3-40ae-8441-14884f9719af\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2cb08497-c1a3-40ae-8441-14884f9719af\\\",\\n \\\"name\\\": \\\"2cb08497-c1a3-40ae-8441-14884f9719af\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-16T12:39:38.97159587Z\\\",\\n \\\"endTime\\\": \\\"2026-03-16T12:41:03.165372775Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-215/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-215/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-----------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"07658b246e4daee088a608885c7327fb84d3a56634579296c63de3996be09a62","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4478/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507756547248128","occurred_at":"2026-03-16T11:36:44Z","failed_at":"e2e","signature_id":"503e71a8f4c408e06b966a9281d228c3dfa630b4b6d0b3f4355de7d564a869c9","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f2c4f8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-55lxx7\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0011f5400\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-55lxx7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0011f53e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-55lxx7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0011f53c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-55lxx7\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f2c4f8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-55lxx7\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0011f5400\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-55lxx7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0011f53e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-55lxx7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0011f53c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-55lxx7\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"44208a35f69811b239317db32ac150539dbff70aaf59d39439c6c6f6a68f2192","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4478/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507756547248128","occurred_at":"2026-03-16T11:36:44Z","failed_at":"e2e","signature_id":"cf66c1f3f958fcb5fd82d374f6763db0cea6c2cc4d3f43cc6c4e7ccf8b241fe8","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001198570\u003e: \n failed to cleanup resource group: failed waiting for resourcegroup=\"e2e-cidr-connectivity-7ftvnr\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000801440\u003e{\n msg: \"failed to cleanup resource group: failed waiting for resourcegroup=\\\"e2e-cidr-connectivity-7ftvnr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000801420\u003e{\n msg: \"failed waiting for resourcegroup=\\\"e2e-cidr-connectivity-7ftvnr\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001198570\u003e: \n failed to cleanup resource group: failed waiting for resourcegroup=\"e2e-cidr-connectivity-7ftvnr\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000801440\u003e{\n msg: \"failed to cleanup resource group: failed waiting for resourcegroup=\\\"e2e-cidr-connectivity-7ftvnr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000801420\u003e{\n msg: \"failed waiting for resourcegroup=\\\"e2e-cidr-connectivity-7ftvnr\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d55c4a29bbbf19a43335bfa2157018ebc82a1c8fe8cbea22f78ef2ca4c570f1a","failure_pattern":"failed to cleanup resource group: failed waiting for resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: failed waiting for resourcegroup=\"autoscaling-cluster-2bvkgc\" to finish deleting: context canceled"}},{"row_id":"470bb2e25bf680b71eb756aeffaf611005208b96df86e7b17d6c55f49c659891","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4478/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507756547248128","occurred_at":"2026-03-16T11:36:44Z","failed_at":"e2e","signature_id":"8e8f21967d25350466631a71dc3c9b2b0070cfe43e90d669352c76256cffeaa1","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ec2858\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-hr9wkd\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000721980\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-hr9wkd\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000721960\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-hr9wkd\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00154e9e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-hr9wkd\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ec2858\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-hr9wkd\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000721980\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-hr9wkd\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000721960\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-hr9wkd\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00154e9e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-hr9wkd\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"4ab633da54f88e7cd07c5d1adda4cc6862ad22bd5a897b72f919d9d23186e50e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4478/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507756547248128","occurred_at":"2026-03-16T11:36:44Z","failed_at":"e2e","signature_id":"826386311e7bea5810e7c24b305508f18d0323e9ffb799f4c18ba26e70f76dec","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001500198\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-9dw6pw\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001802e40\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-9dw6pw\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001802e20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-9dw6pw\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001802e00\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-9dw6pw\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001500198\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-9dw6pw\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001802e40\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-9dw6pw\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001802e20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-9dw6pw\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001802e00\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-9dw6pw\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"500f942ad987bbeccde162d452c32176bfc26042cb3c17b5bc50bd2d88b7d240","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4478/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507756547248128","occurred_at":"2026-03-16T11:36:44Z","failed_at":"e2e","signature_id":"45a4f279b6bbfe3d520d16c6dfe0b20e6259552dd05c5000b4ff813263bc0012","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00124c1e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-x9wqmp\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0005a1ee0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-x9wqmp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0005a1ec0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-x9wqmp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0005a1ea0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-x9wqmp\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00124c1e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-x9wqmp\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0005a1ee0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-x9wqmp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0005a1ec0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-x9wqmp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0005a1ea0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-x9wqmp\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"6bee3fa1dd9cb4f4314bbbd2e12356c8cd706ea06376048a9f11854907626705","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4478/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507756547248128","occurred_at":"2026-03-16T11:36:44Z","failed_at":"e2e","signature_id":"aeb0371181bfac1e72416697469a61808a438b05bb329e2ee3efbc03f06651bf","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000eee450\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-c45gh9\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001193220\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-c45gh9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001193200\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-c45gh9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0011931e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-c45gh9\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000eee450\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-c45gh9\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001193220\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-c45gh9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001193200\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-c45gh9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0011931e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-c45gh9\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"77382c0a33f649d3a06d4974cb645b244c336225bde4708d98874cee4398bfca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4478/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507756547248128","occurred_at":"2026-03-16T11:36:44Z","failed_at":"e2e","signature_id":"e36249b8dfe10815fabf300ce009030c2850efca8104d7a91bc74cf3ed4a341b","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00079b260\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-l2plxm\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000709160\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-l2plxm\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000709140\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-l2plxm\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000709120\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-l2plxm\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00079b260\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-l2plxm\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000709160\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-l2plxm\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000709140\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-l2plxm\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000709120\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-l2plxm\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"86bd02636208d561d177a3a2d75b63724fb95c2dbe17187f00f467e8d3aae45c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4478/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507756547248128","occurred_at":"2026-03-16T11:36:44Z","failed_at":"e2e","signature_id":"822c472875ba14760b7dbbb85f1c6ecd9baf157913e99cc0ad0d884c82af1265","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000fc0330\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-psb9ww\" in resourcegroup=\"admin-credential-lifecycle-test-9jj5q7\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000b8b2e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-psb9ww\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-9jj5q7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b8b2c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-psb9ww\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-9jj5q7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b8b2a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-psb9ww\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-9jj5q7\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000fc0330\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-psb9ww\" in resourcegroup=\"admin-credential-lifecycle-test-9jj5q7\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000b8b2e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-psb9ww\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-9jj5q7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b8b2c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-psb9ww\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-9jj5q7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b8b2a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-psb9ww\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-9jj5q7\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"892f91acd602f3be65fd4bce9d2d39dea495e63214f479a203d641e4b5ba0db3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4478/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507756547248128","occurred_at":"2026-03-16T11:36:44Z","failed_at":"e2e","signature_id":"10bde12d297a901154f501f29072f576cfc96dcd5e81bc047ce2ffde9904bbfb","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00139c2d0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-q94kvk\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0004f68e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-q94kvk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0004f68c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-q94kvk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0004f68a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-q94kvk\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00139c2d0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-q94kvk\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0004f68e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-q94kvk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0004f68c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-q94kvk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0004f68a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-q94kvk\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"96811dd7cf317b24ca4b908cf7980c209e5fa27248add0542ec1485086fd5e91","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4478/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507756547248128","occurred_at":"2026-03-16T11:36:44Z","failed_at":"e2e","signature_id":"361652880027a93710e64a74efb00e55e21d4f5070400cbb67189877be87e3de","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f38198\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-85cpq6\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000b7eec0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-85cpq6\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b7eea0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-85cpq6\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b7ee80\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-85cpq6\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f38198\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-85cpq6\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000b7eec0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-85cpq6\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b7eea0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-85cpq6\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b7ee80\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-85cpq6\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"992280c334c7c9eaeae893bf6383a45f2b7a810aca0532964a8144d7289a178c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4478/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507756547248128","occurred_at":"2026-03-16T11:36:44Z","failed_at":"e2e","signature_id":"5a1a1ed0fc8b8c3ab53cc9cc05f2d1c1abc2df3fd0aa7023e0dce748356eeacb","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.006s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0011e4a70\u003e: \n missing expected log sources: [ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [ocm-arohcp/kube-apiserver]\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.006s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0011e4a70\u003e: \n missing expected log sources: [ocm-arohcp/kube-apiserver]\n {\n s: \"missing expected log sources: [ocm-arohcp/kube-apiserver]\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d68a3a4000afced929cc779e7a36af528e5b7a9739bb3b6369253f0a6d739eb2","failure_pattern":"missing expected log sources: [ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [ocm-arohcp/kube-apiserver]"}},{"row_id":"a769ace7ed28d608db01a7054f32ba3bb6c0e19516849f4844bdd7f60cb86b51","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4478/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507756547248128","occurred_at":"2026-03-16T11:36:44Z","failed_at":"e2e","signature_id":"1a4eb5a68b713cd59c6f0509f48454cdb3693c70fdeb05076e4c96bb87818d27","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000bf4c30\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-4bgn2f\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f2a180\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-4bgn2f\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f2a160\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-4bgn2f\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f2a140\u003e{\n msg: \"failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-4bgn2f\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000bf4c30\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-4bgn2f\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f2a180\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-4bgn2f\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f2a160\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-4bgn2f\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f2a140\u003e{\n msg: \"failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-4bgn2f\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"b8e120667da5533037dbe70b00b2e39acb7e93da989de394a8b728a64a157c53","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4478/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507756547248128","occurred_at":"2026-03-16T11:36:44Z","failed_at":"e2e","signature_id":"8a7e69bebc2ecca3f3e3e980883f8aefb42d1f420393da50d8d3e5cb2cb1a899","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00176a198\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-qm2lgz\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00090dc00\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-qm2lgz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00090dbe0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-qm2lgz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00090dbc0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-qm2lgz\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00176a198\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-qm2lgz\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00090dc00\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-qm2lgz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00090dbe0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-qm2lgz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00090dbc0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-qm2lgz\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}},{"row_id":"d66300f104fd6b6751c827083925b7fca4eabd8665212aa0acbcaff60cb3241f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4478/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507756547248128","occurred_at":"2026-03-16T11:36:44Z","failed_at":"e2e","signature_id":"036bd176c82725d264335afee4e5a191f62373d9dc97907dd13db48ba1273fb3","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0013bc450\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-k2vtm5\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a1a440\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-k2vtm5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a1a420\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-k2vtm5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a1a400\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-k2vtm5\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0013bc450\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-k2vtm5\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a1a440\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-k2vtm5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a1a420\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-k2vtm5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a1a400\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-k2vtm5\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0980c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-22nlpz\" in resourcegroup=\"admin-credential-lifecycle-test-69cfx2\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":15,"failure_pattern_ids":["503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","d55c4a29bbbf19a43335bfa2157018ebc82a1c8fe8cbea22f78ef2ca4c570f1a","d68a3a4000afced929cc779e7a36af528e5b7a9739bb3b6369253f0a6d739eb2"],"matched_occurrences":15,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507724687314944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4479,"pr_state":"open","pr_sha":"6b0d466327c0f476bad68a00bbf7fe9e391a4fb3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:36:36Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b3279374736653a534b13986ee536ac7749ec4c91ca1f18663c777976d5b4525","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033507724687314944","occurred_at":"2026-03-16T11:36:36Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033498763032530944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4117,"pr_state":"closed","pr_sha":"2eda04857baa3d253d3cc30afc7eafe330c6b6b2","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T11:00:39Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"3baede0645d62b1a586d1f65816985c9f1bdccb04c5ace139e9bb6e9496ea524","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033498763032530944","occurred_at":"2026-03-16T11:00:39Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033495777363365888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4479,"pr_state":"open","pr_sha":"6b0d466327c0f476bad68a00bbf7fe9e391a4fb3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T10:49:08Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"87f8ac5ec6307030e1078abad1262d984f60669f4907ba3d21b168dc38dbb4e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033495777363365888","occurred_at":"2026-03-16T10:49:08Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033484306701094912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4479,"pr_state":"open","pr_sha":"ec4e7390d1256ab51fc7dc925222fd702f996cfd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T10:03:33Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"f81ae4795e28597989f69673cf512d7a75820f25193110022475b0de64bd3fc8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033484306701094912","occurred_at":"2026-03-16T10:03:33Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033459707510788096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4347,"pr_state":"closed","pr_sha":"a82c62541bf569a06efb0a6211c6ae9e8a2c01a4","final_merged_sha":"a82c62541bf569a06efb0a6211c6ae9e8a2c01a4","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-16T08:25:48Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4449/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033431377453518848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4449,"pr_state":"closed","pr_sha":"b7f1a9cc9a3f368df471da560700df1758cf283d","final_merged_sha":"205984e6ef0d1b76b1e624468408854ec2a65a8c","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-16T06:33:14Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4188/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033429730136428544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4188,"pr_state":"closed","pr_sha":"c45e5a90356f27c66b7010a7081be7a8eb034202","final_merged_sha":"ad7003e5a941946bb5a1fed27c539b6ee3d95872","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-16T06:26:41Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033416459568287744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4477,"pr_state":"open","pr_sha":"79968855b403781872aa32d09e475edd3cbcd658","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T05:33:57Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ec31e5ffe00c71a7c9d114d2188d43f468abdf7d69a39e4bfe4ee99131315e16","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033416459568287744","occurred_at":"2026-03-16T05:33:57Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4476/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033407602041294848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4476,"pr_state":"closed","pr_sha":"c98393e210b23f8d527e003a5c862b5c0602e621","final_merged_sha":"c98393e210b23f8d527e003a5c862b5c0602e621","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-16T04:58:45Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033384666504892416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4473,"pr_state":"closed","pr_sha":"bec5815ca7dc3703fc750e5f2ef845414e297409","final_merged_sha":"8b7576c81d01104dc238546ec2678952bf7afd61","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T03:27:37Z"},"failed_at":["e2e"],"failed_test_count":25,"occurrences":[{"row_id":"0400e7133e80e434ae6f6ee06142813e56f1d5557e27f6538e29f500ed8af1eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033384666504892416","occurred_at":"2026-03-16T03:27:37Z","failed_at":"e2e","signature_id":"f97b674f7b838449ad12a94411d4a461874ac3ac80036f44d391f45819c7fee5","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00144fea0\u003e: \n failed to create HCP cluster ea-listxtfzbl: failed starting cluster creation \"ea-listxtfzbl\" in resourcegroup=\"ea-list-jfmwrk\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-jfmwrk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listxtfzbl\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster ea-listxtfzbl: failed starting cluster creation \\\"ea-listxtfzbl\\\" in resourcegroup=\\\"ea-list-jfmwrk\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-jfmwrk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listxtfzbl\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00144fe80\u003e{\n msg: \"failed starting cluster creation \\\"ea-listxtfzbl\\\" in resourcegroup=\\\"ea-list-jfmwrk\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-jfmwrk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listxtfzbl\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001700390\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"8cbaaac2-5d0d-4177-9627-dcb9c5fe7dcf\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:15:09 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"31\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00144fdc0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 50, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00144fea0\u003e: \n failed to create HCP cluster ea-listxtfzbl: failed starting cluster creation \"ea-listxtfzbl\" in resourcegroup=\"ea-list-jfmwrk\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-jfmwrk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listxtfzbl\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster ea-listxtfzbl: failed starting cluster creation \\\"ea-listxtfzbl\\\" in resourcegroup=\\\"ea-list-jfmwrk\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-jfmwrk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listxtfzbl\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00144fe80\u003e{\n msg: \"failed starting cluster creation \\\"ea-listxtfzbl\\\" in resourcegroup=\\\"ea-list-jfmwrk\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-jfmwrk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listxtfzbl\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001700390\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"8cbaaac2-5d0d-4177-9627-dcb9c5fe7dcf\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:15:09 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"31\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00144fdc0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 50, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"401bf9e291581002825780df3fb4b7e53257787fe51de289cb4d70bdb3f7a9da","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Unsupported value: \"\": supported values: \"Private; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"0fdb2a2c937fc7049bb43c0607bbd7f49e761e2d085a516b77e480d831197197","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033384666504892416","occurred_at":"2026-03-16T03:27:37Z","failed_at":"e2e","signature_id":"ee5260e106865fbed8e4500bc6be508ce93752664114334e4a81ae5c68b03c5a","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0005b1ac0\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-6xn8gb\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-6xn8gb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-6xn8gb\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-6xn8gb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0005b1aa0\u003e{\n msg: \"failed starting cluster creation \\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-6xn8gb\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-6xn8gb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0011bf650\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"3e212603-e8a5-4c0a-abcc-2ae40bf4b97c\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:15:52 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"34\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0005b19e0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 3...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0005b1ac0\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-6xn8gb\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-6xn8gb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-6xn8gb\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-6xn8gb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0005b1aa0\u003e{\n msg: \"failed starting cluster creation \\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-6xn8gb\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-6xn8gb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0011bf650\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"3e212603-e8a5-4c0a-abcc-2ae40bf4b97c\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:15:52 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"34\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0005b19e0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 3...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"401bf9e291581002825780df3fb4b7e53257787fe51de289cb4d70bdb3f7a9da","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Unsupported value: \"\": supported values: \"Private; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"13259c32711769557be7afd7b316eec97a32125fbc89bb0237d47dc56c03d855","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033384666504892416","occurred_at":"2026-03-16T03:27:37Z","failed_at":"e2e","signature_id":"0aa4ed2b99c8f955699fd8eceb556ee5509fab540327ca70261c47ffe7a21e8f","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0013d47c0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-j7v8j4\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-j7v8j4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-j7v8j4\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-j7v8j4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0013d47a0\u003e{\n msg: \"failed starting cluster creation \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-j7v8j4\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-j7v8j4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000e72510\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Envoy-Upstream-Service-Time\": [\"32\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"3903f69a-5bf4-4b44-afcd-b5a699f79ba1\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:14:59 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0013d46e0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0013d47c0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-j7v8j4\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-j7v8j4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-j7v8j4\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-j7v8j4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0013d47a0\u003e{\n msg: \"failed starting cluster creation \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-j7v8j4\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-j7v8j4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000e72510\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Envoy-Upstream-Service-Time\": [\"32\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"3903f69a-5bf4-4b44-afcd-b5a699f79ba1\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:14:59 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0013d46e0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"401bf9e291581002825780df3fb4b7e53257787fe51de289cb4d70bdb3f7a9da","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Unsupported value: \"\": supported values: \"Private; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"195bcec19ea994f70f24ecfefce34b31e8a4b22553f07c92e8c42754c455d17b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033384666504892416","occurred_at":"2026-03-16T03:27:37Z","failed_at":"e2e","signature_id":"3c690e1ecc6ff36c5b4d95fac182c15195e83496b317008a430272b8bc84c61a","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001026700\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-bvplml\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-bvplml/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-bvplml\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-bvplml/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0010266e0\u003e{\n msg: \"failed starting cluster creation \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-bvplml\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-bvplml/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001501e90\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"2dd0563d-27e8-4b59-a5d6-c0ae1ec5ecd7\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:15:10 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"30\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc001026620\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001026700\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-bvplml\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-bvplml/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-bvplml\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-bvplml/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0010266e0\u003e{\n msg: \"failed starting cluster creation \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-bvplml\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-bvplml/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001501e90\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"2dd0563d-27e8-4b59-a5d6-c0ae1ec5ecd7\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:15:10 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"30\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc001026620\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"401bf9e291581002825780df3fb4b7e53257787fe51de289cb4d70bdb3f7a9da","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Unsupported value: \"\": supported values: \"Private; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"283ef0fdd2e66c6fbccdc062e3ffc422e15a4f1a5ff3397c41e66e4c1ed8afb6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033384666504892416","occurred_at":"2026-03-16T03:27:37Z","failed_at":"e2e","signature_id":"afb40ebfeb0b9bb419d45362e72c747ee7a73667d6a56189bdb93a7a853b4051","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00082bb20\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-vcpd4k\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-vcpd4k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-vcpd4k\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-vcpd4k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00082bb00\u003e{\n msg: \"failed starting cluster creation \\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-vcpd4k\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-vcpd4k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000bc4d80\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"a83e4eff-6581-4d57-8525-7b4553b46f8a\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:18:02 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"32\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00082ba40\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00082bb20\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-vcpd4k\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-vcpd4k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-vcpd4k\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-vcpd4k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00082bb00\u003e{\n msg: \"failed starting cluster creation \\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-vcpd4k\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-vcpd4k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000bc4d80\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"a83e4eff-6581-4d57-8525-7b4553b46f8a\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:18:02 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"32\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00082ba40\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"401bf9e291581002825780df3fb4b7e53257787fe51de289cb4d70bdb3f7a9da","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Unsupported value: \"\": supported values: \"Private; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"3a51a238db92c6aff223568df49287ef4500ca60a6247d5c392dde183b485a65","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033384666504892416","occurred_at":"2026-03-16T03:27:37Z","failed_at":"e2e","signature_id":"eb7348146561fbe34575e3ac36584a497b36ea56ad9f57ed50ce17b2852b0b8c","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:96]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0010a5380\u003e: \n failed to create HCP cluster basic-hcp-cluster: failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-kr6qd2\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-kr6qd2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster basic-hcp-cluster: failed starting cluster creation \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-kr6qd2\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-kr6qd2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0010a5360\u003e{\n msg: \"failed starting cluster creation \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-kr6qd2\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-kr6qd2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00109f920\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"ec42b69e-3ae1-4087-9fc5-1ae1e63e8c3b\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:14:38 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"42\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0010a52a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:96]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0010a5380\u003e: \n failed to create HCP cluster basic-hcp-cluster: failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-kr6qd2\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-kr6qd2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster basic-hcp-cluster: failed starting cluster creation \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-kr6qd2\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-kr6qd2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0010a5360\u003e{\n msg: \"failed starting cluster creation \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-kr6qd2\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-kr6qd2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00109f920\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"ec42b69e-3ae1-4087-9fc5-1ae1e63e8c3b\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:14:38 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"42\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0010a52a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"401bf9e291581002825780df3fb4b7e53257787fe51de289cb4d70bdb3f7a9da","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Unsupported value: \"\": supported values: \"Private; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"4b126407f9c90d8d51485f2c895da94c2b8e4ab8fa84d06bb346f611db683e0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033384666504892416","occurred_at":"2026-03-16T03:27:37Z","failed_at":"e2e","signature_id":"3ce6eb3418b9c802d12a3660c72bd0062adaea0fcb6de6ae29cf4ff4f55cdd6a","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00100a6c0\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed starting cluster creation \"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-zrxld6\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-zrxld6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/sre-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster sre-hcp-cluster: failed starting cluster creation \\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-zrxld6\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-zrxld6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/sre-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00100a6a0\u003e{\n msg: \"failed starting cluster creation \\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-zrxld6\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-zrxld6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/sre-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000eade90\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Envoy-Upstream-Service-Time\": [\"1036\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"f2fd150a-d81c-4682-abff-782da480d223\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:15:00 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00100a5e0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00100a6c0\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed starting cluster creation \"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-zrxld6\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-zrxld6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/sre-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster sre-hcp-cluster: failed starting cluster creation \\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-zrxld6\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-zrxld6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/sre-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00100a6a0\u003e{\n msg: \"failed starting cluster creation \\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-zrxld6\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-zrxld6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/sre-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000eade90\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Envoy-Upstream-Service-Time\": [\"1036\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"f2fd150a-d81c-4682-abff-782da480d223\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:15:00 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00100a5e0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"401bf9e291581002825780df3fb4b7e53257787fe51de289cb4d70bdb3f7a9da","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Unsupported value: \"\": supported values: \"Private; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"4be54273edfaab91bf7fe10df4b9ec312071fb63ebc76eddeb978647b7661fb0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033384666504892416","occurred_at":"2026-03-16T03:27:37Z","failed_at":"e2e","signature_id":"4fa0a46b308a04232fb0664c72b661bd49588c1a5802f37d37f8429d1e741c59","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0006f8c00\u003e: \n failed starting cluster creation \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-jhvh92\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-jhvh92/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed starting cluster creation \\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-jhvh92\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-jhvh92/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0015190b0\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Mon, 16 Mar 2026 04:14:51 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"1037\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"da5ab60c-4711-4b7f-905e-d8a02d0113ae\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0006f8aa0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 50, 32, 101, 114, 114, 111, 114, 115, 32, 111, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 100, 101, 116, 97, 105, 108, 115, 34, 58, 32, 91, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 82, 101, 113, 117, 105, 114, 101, 100, 32, 118, 97, 108, 117, 101, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 112, 114, 111, 112, 101, 114, 116, 105, 101, 115, 46, 101, 116, 99, 100, 46, 100, 97, 116, 97, 69, 110, 99, 114, 121, 112, 116, 105, 111, 110, 46, 99, 117, 115, 116, 111, 109, 101, 114, 77, 97, 110, 97, 103, 101, 100, 46, 107, 109, 115, 46, 118, 105, 115, 105, 98, 105, 108, 105, 116, 121, 34, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 125, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 1...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0006f8c00\u003e: \n failed starting cluster creation \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-jhvh92\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-jhvh92/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed starting cluster creation \\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-jhvh92\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-jhvh92/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0015190b0\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Mon, 16 Mar 2026 04:14:51 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"1037\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"da5ab60c-4711-4b7f-905e-d8a02d0113ae\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0006f8aa0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 50, 32, 101, 114, 114, 111, 114, 115, 32, 111, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 100, 101, 116, 97, 105, 108, 115, 34, 58, 32, 91, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 82, 101, 113, 117, 105, 114, 101, 100, 32, 118, 97, 108, 117, 101, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 112, 114, 111, 112, 101, 114, 116, 105, 101, 115, 46, 101, 116, 99, 100, 46, 100, 97, 116, 97, 69, 110, 99, 114, 121, 112, 116, 105, 111, 110, 46, 99, 117, 115, 116, 111, 109, 101, 114, 77, 97, 110, 97, 103, 101, 100, 46, 107, 109, 115, 46, 118, 105, 115, 105, 98, 105, 108, 105, 116, 121, 34, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 125, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 1...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"401bf9e291581002825780df3fb4b7e53257787fe51de289cb4d70bdb3f7a9da","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Unsupported value: \"\": supported values: \"Private; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"4d077f4f1d7e9824354ad27b98c8c70e7fee0663969acd2228c04edd9be053cb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033384666504892416","occurred_at":"2026-03-16T03:27:37Z","failed_at":"e2e","signature_id":"125e5bbbdac0bd2a03797288d3241a8ba26e9ea38ee9c5dfe33dcc84063b25ad","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00077fd80\u003e: \n failed to create HCP cluster list-test-cluster-l7lj2f: failed starting cluster creation \"list-test-cluster-l7lj2f\" in resourcegroup=\"cluster-listing-zpd4tf\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-zpd4tf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-l7lj2f\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster list-test-cluster-l7lj2f: failed starting cluster creation \\\"list-test-cluster-l7lj2f\\\" in resourcegroup=\\\"cluster-listing-zpd4tf\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-zpd4tf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-l7lj2f\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00077fd60\u003e{\n msg: \"failed starting cluster creation \\\"list-test-cluster-l7lj2f\\\" in resourcegroup=\\\"cluster-listing-zpd4tf\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-zpd4tf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-l7lj2f\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0011b6f30\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Mon, 16 Mar 2026 04:14:49 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"40\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"44c2886e-d220-4d87-b3ed-5f07c57742a2\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00077fc80\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00077fd80\u003e: \n failed to create HCP cluster list-test-cluster-l7lj2f: failed starting cluster creation \"list-test-cluster-l7lj2f\" in resourcegroup=\"cluster-listing-zpd4tf\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-zpd4tf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-l7lj2f\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster list-test-cluster-l7lj2f: failed starting cluster creation \\\"list-test-cluster-l7lj2f\\\" in resourcegroup=\\\"cluster-listing-zpd4tf\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-zpd4tf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-l7lj2f\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00077fd60\u003e{\n msg: \"failed starting cluster creation \\\"list-test-cluster-l7lj2f\\\" in resourcegroup=\\\"cluster-listing-zpd4tf\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-zpd4tf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-l7lj2f\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0011b6f30\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Mon, 16 Mar 2026 04:14:49 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"40\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"44c2886e-d220-4d87-b3ed-5f07c57742a2\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00077fc80\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"401bf9e291581002825780df3fb4b7e53257787fe51de289cb4d70bdb3f7a9da","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Unsupported value: \"\": supported values: \"Private; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"516967c771a8dff61b216e4bc322605089433f8b8b448579007e4b5013cee281","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033384666504892416","occurred_at":"2026-03-16T03:27:37Z","failed_at":"e2e","signature_id":"85c559b049c3942dd5f4494509c4db6ba38b1c15d3cdecc4a6fd6a33e07b5c16","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0005b1060\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed starting cluster creation \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-j2xt9k\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-j2xt9k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster engineering-hcp-cluster: failed starting cluster creation \\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-j2xt9k\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-j2xt9k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0005b1040\u003e{\n msg: \"failed starting cluster creation \\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-j2xt9k\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-j2xt9k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012982d0\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Mon, 16 Mar 2026 04:17:57 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"32\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"7a38c7af-7c8e-43db-90f9-c06ae146cc50\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0005b0ba0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0005b1060\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed starting cluster creation \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-j2xt9k\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-j2xt9k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster engineering-hcp-cluster: failed starting cluster creation \\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-j2xt9k\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-j2xt9k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0005b1040\u003e{\n msg: \"failed starting cluster creation \\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-j2xt9k\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-j2xt9k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012982d0\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Mon, 16 Mar 2026 04:17:57 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"32\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"7a38c7af-7c8e-43db-90f9-c06ae146cc50\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0005b0ba0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"401bf9e291581002825780df3fb4b7e53257787fe51de289cb4d70bdb3f7a9da","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Unsupported value: \"\": supported values: \"Private; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"66ee3a924c8e58363ac9c044c7ed9a5bb99e9da9e10477f461c1ac5af4ab318c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033384666504892416","occurred_at":"2026-03-16T03:27:37Z","failed_at":"e2e","signature_id":"f64bea01814f46e231e637cef4c75c350d5aca83ccb285432c1ca702c5b56a6d","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000cf90a0\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed starting cluster creation \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-p2p9tv\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-p2p9tv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster: failed starting cluster creation \\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-p2p9tv\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-p2p9tv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000cf9080\u003e{\n msg: \"failed starting cluster creation \\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-p2p9tv\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-p2p9tv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0014f8b10\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"6c8730ae-074d-4143-88b0-ae8b390dc6ab\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:18:12 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"30\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000cf8fc0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 1...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000cf90a0\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed starting cluster creation \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-p2p9tv\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-p2p9tv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster: failed starting cluster creation \\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-p2p9tv\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-p2p9tv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000cf9080\u003e{\n msg: \"failed starting cluster creation \\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-p2p9tv\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-p2p9tv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0014f8b10\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"6c8730ae-074d-4143-88b0-ae8b390dc6ab\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:18:12 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"30\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000cf8fc0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 1...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"401bf9e291581002825780df3fb4b7e53257787fe51de289cb4d70bdb3f7a9da","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Unsupported value: \"\": supported values: \"Private; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"7af3d0df5aebcf2a31c584128f8401673bbc427d8c186f75d5433ab01a9099ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033384666504892416","occurred_at":"2026-03-16T03:27:37Z","failed_at":"e2e","signature_id":"3ad4f225d017623a0b1eb89d213f2b14c47d375f91cf93b439fbcb5d2dfa08c6","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00012f740\u003e: \n failed to create HCP cluster cluster-zstream-4-20-plc7k8: failed starting cluster creation \"cluster-zstream-4-20-plc7k8\" in resourcegroup=\"rg-zstream-upgrade-4-20-mqn7nb\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-mqn7nb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-plc7k8\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-20-plc7k8: failed starting cluster creation \\\"cluster-zstream-4-20-plc7k8\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-mqn7nb\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-mqn7nb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-plc7k8\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00012f720\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-20-plc7k8\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-mqn7nb\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-mqn7nb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-plc7k8\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000a960f0\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"31\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"f106c0f1-5ffe-45ed-a98f-64baa6997ef9\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:18:02 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00012ef40\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 9...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00012f740\u003e: \n failed to create HCP cluster cluster-zstream-4-20-plc7k8: failed starting cluster creation \"cluster-zstream-4-20-plc7k8\" in resourcegroup=\"rg-zstream-upgrade-4-20-mqn7nb\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-mqn7nb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-plc7k8\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-20-plc7k8: failed starting cluster creation \\\"cluster-zstream-4-20-plc7k8\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-mqn7nb\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-mqn7nb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-plc7k8\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00012f720\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-20-plc7k8\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-mqn7nb\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-mqn7nb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-plc7k8\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000a960f0\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"31\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"f106c0f1-5ffe-45ed-a98f-64baa6997ef9\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:18:02 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00012ef40\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 9...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"401bf9e291581002825780df3fb4b7e53257787fe51de289cb4d70bdb3f7a9da","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Unsupported value: \"\": supported values: \"Private; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"9a19a117c589b0706aa0e8062e5b7effbd0703b31f94cf3b5aed2027ba8873ae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033384666504892416","occurred_at":"2026-03-16T03:27:37Z","failed_at":"e2e","signature_id":"df9c00fcc5420b9175e108aa846b4735175d3463a1b1e7fa2f92dae5169738db","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:89]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e10980\u003e: \n failed to create HCP cluster negative-tests-cluster: failed starting cluster creation \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-zfrg75\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-zfrg75/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster negative-tests-cluster: failed starting cluster creation \\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-zfrg75\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-zfrg75/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e10960\u003e{\n msg: \"failed starting cluster creation \\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-zfrg75\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-zfrg75/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000f8f350\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Mon, 16 Mar 2026 04:16:54 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"35\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"378d7b05-d9f3-42e0-b072-96ff2fabaabb\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000e108a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 3...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:89]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e10980\u003e: \n failed to create HCP cluster negative-tests-cluster: failed starting cluster creation \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-zfrg75\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-zfrg75/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster negative-tests-cluster: failed starting cluster creation \\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-zfrg75\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-zfrg75/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e10960\u003e{\n msg: \"failed starting cluster creation \\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-zfrg75\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-zfrg75/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000f8f350\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Mon, 16 Mar 2026 04:16:54 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"35\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"378d7b05-d9f3-42e0-b072-96ff2fabaabb\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000e108a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 3...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"401bf9e291581002825780df3fb4b7e53257787fe51de289cb4d70bdb3f7a9da","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Unsupported value: \"\": supported values: \"Private; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"a4ae3650a8356c255be84adfaf9d2132d0bbfc22f065597520a5313a05eaaff7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033384666504892416","occurred_at":"2026-03-16T03:27:37Z","failed_at":"e2e","signature_id":"d7a03997d380b1f13c89edffd8eaaf9fee24ddf1984c243329b24bd30fc234ce","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00096cd00\u003e: \n failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \"patch-tags-cluster\" in resourcegroup=\"patch-tags-fftqnz\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-fftqnz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-fftqnz\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-fftqnz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00096cce0\u003e{\n msg: \"failed starting cluster creation \\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-fftqnz\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-fftqnz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001210960\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Mon, 16 Mar 2026 04:14:59 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"32\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"559624b8-e5e1-4ac4-b328-25a6988fe090\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00096cc20\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 11...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00096cd00\u003e: \n failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \"patch-tags-cluster\" in resourcegroup=\"patch-tags-fftqnz\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-fftqnz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-fftqnz\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-fftqnz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00096cce0\u003e{\n msg: \"failed starting cluster creation \\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-fftqnz\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-fftqnz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001210960\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Mon, 16 Mar 2026 04:14:59 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"32\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"559624b8-e5e1-4ac4-b328-25a6988fe090\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00096cc20\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 11...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"401bf9e291581002825780df3fb4b7e53257787fe51de289cb4d70bdb3f7a9da","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Unsupported value: \"\": supported values: \"Private; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"a74e9c598c11331cf722f5449fc1ba984ee9db7b89cc25d4d01754ac416a37bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033384666504892416","occurred_at":"2026-03-16T03:27:37Z","failed_at":"e2e","signature_id":"96664afbb7b1cc111416710c185ea174f6aa9c5bcfb26c5424c49dd1228c4d58","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:86]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b03f20\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-xvv4s9\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-xvv4s9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-xvv4s9\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-xvv4s9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000b03f00\u003e{\n msg: \"failed starting cluster creation \\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-xvv4s9\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-xvv4s9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000ab62a0\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Request-Id\": [\n \"6fef2e8f-635d-46e9-8c86-eac613c46a08\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:17:56 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"30\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000b03d20\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:86]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b03f20\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-xvv4s9\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-xvv4s9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-xvv4s9\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-xvv4s9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000b03f00\u003e{\n msg: \"failed starting cluster creation \\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-xvv4s9\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-xvv4s9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000ab62a0\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Request-Id\": [\n \"6fef2e8f-635d-46e9-8c86-eac613c46a08\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:17:56 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"30\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000b03d20\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"401bf9e291581002825780df3fb4b7e53257787fe51de289cb4d70bdb3f7a9da","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Unsupported value: \"\": supported values: \"Private; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"ad8b6ec0d73baf6281c9d5064d605719c732d297f5219aab7270dde14ca32b56","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033384666504892416","occurred_at":"2026-03-16T03:27:37Z","failed_at":"e2e","signature_id":"3463481981c2724b4dd250382837d17131c4a4f9446ad48fb3b8948d09d559fe","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a89940\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-hp8mjn\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-hp8mjn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-hp8mjn\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-hp8mjn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a89920\u003e{\n msg: \"failed starting cluster creation \\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-hp8mjn\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-hp8mjn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0010226c0\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"29\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"72b59c9f-5470-47b7-8095-dabe5e6a5c8b\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:14:50 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000a89860\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a89940\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-hp8mjn\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-hp8mjn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-hp8mjn\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-hp8mjn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a89920\u003e{\n msg: \"failed starting cluster creation \\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-hp8mjn\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-hp8mjn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0010226c0\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"29\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"72b59c9f-5470-47b7-8095-dabe5e6a5c8b\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:14:50 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000a89860\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"401bf9e291581002825780df3fb4b7e53257787fe51de289cb4d70bdb3f7a9da","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Unsupported value: \"\": supported values: \"Private; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"b294376c909243e3a4e1c90d68ff3fd15c102225e41d97acacf6315771f16b44","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033384666504892416","occurred_at":"2026-03-16T03:27:37Z","failed_at":"e2e","signature_id":"0afd875f1a9256f89288df94f4796c5dea5f441425baf60cb05bb9d6633bf3a1","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:99]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000776520\u003e: \n failed starting cluster creation \"admin-cred-lifecycle-qc4566\" in resourcegroup=\"admin-credential-lifecycle-test-rs9zpk\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-rs9zpk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-qc4566\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed starting cluster creation \\\"admin-cred-lifecycle-qc4566\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-rs9zpk\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-rs9zpk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-qc4566\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0015fe030\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Mon, 16 Mar 2026 04:14:59 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"40\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"8a57294a-cd16-48d8-bcf8-086bec0a9db0\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0007763e0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 50, 32, 101, 114, 114, 111, 114, 115, 32, 111, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 100, 101, 116, 97, 105, 108, 115, 34, 58, 32, 91, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 82, 101, 113, 117, 105, 114, 101, 100, 32, 118, 97, 108, 117, 101, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 112, 114, 111, 112, 101, 114, 116, 105, 101, 115, 46, 101, 116, 99, 100, 46, 100, 97, 116, 97, 69, 110, 99, 114, 121, 112, 116, 105, 111, 110, 46, 99, 117, 115, 116, 111, 109, 101, 114, 77, 97, 110, 97, 103, 101, 100, 46, 107, 109, 115, 46, 118, 105, 115, 105, 98, 105, 108, 105, 116, 121, 34, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 125, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:99]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000776520\u003e: \n failed starting cluster creation \"admin-cred-lifecycle-qc4566\" in resourcegroup=\"admin-credential-lifecycle-test-rs9zpk\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-rs9zpk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-qc4566\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed starting cluster creation \\\"admin-cred-lifecycle-qc4566\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-rs9zpk\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-rs9zpk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-qc4566\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0015fe030\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Mon, 16 Mar 2026 04:14:59 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"40\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"8a57294a-cd16-48d8-bcf8-086bec0a9db0\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0007763e0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 50, 32, 101, 114, 114, 111, 114, 115, 32, 111, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 100, 101, 116, 97, 105, 108, 115, 34, 58, 32, 91, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 82, 101, 113, 117, 105, 114, 101, 100, 32, 118, 97, 108, 117, 101, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 112, 114, 111, 112, 101, 114, 116, 105, 101, 115, 46, 101, 116, 99, 100, 46, 100, 97, 116, 97, 69, 110, 99, 114, 121, 112, 116, 105, 111, 110, 46, 99, 117, 115, 116, 111, 109, 101, 114, 77, 97, 110, 97, 103, 101, 100, 46, 107, 109, 115, 46, 118, 105, 115, 105, 98, 105, 108, 105, 116, 121, 34, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 125, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"401bf9e291581002825780df3fb4b7e53257787fe51de289cb4d70bdb3f7a9da","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Unsupported value: \"\": supported values: \"Private; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"b906fd0736c786173aa9f38e2605260371c168b55262d13ef997fe43a19b4475","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033384666504892416","occurred_at":"2026-03-16T03:27:37Z","failed_at":"e2e","signature_id":"ecfcd35722335c6ed185f77947f4a1e249d672a22cd68e2669b8d15d9cd4d0ab","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00073f740\u003e: \n failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-v6hrpd\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-v6hrpd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster basic-cluster: failed starting cluster creation \\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-v6hrpd\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-v6hrpd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00073f700\u003e{\n msg: \"failed starting cluster creation \\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-v6hrpd\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-v6hrpd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012e6b70\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Mon, 16 Mar 2026 04:15:00 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"32\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"243ff774-2aae-4b08-a595-4aaf80f350ef\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00073f540\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00073f740\u003e: \n failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-v6hrpd\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-v6hrpd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster basic-cluster: failed starting cluster creation \\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-v6hrpd\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-v6hrpd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00073f700\u003e{\n msg: \"failed starting cluster creation \\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-v6hrpd\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-v6hrpd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012e6b70\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Mon, 16 Mar 2026 04:15:00 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"32\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"243ff774-2aae-4b08-a595-4aaf80f350ef\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00073f540\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"401bf9e291581002825780df3fb4b7e53257787fe51de289cb4d70bdb3f7a9da","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Unsupported value: \"\": supported values: \"Private; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"be348d497b39a74bc65ecebe216e64b6f248167c50626083cba9d31f8039755f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033384666504892416","occurred_at":"2026-03-16T03:27:37Z","failed_at":"e2e","signature_id":"85359ed2fe4ee38bbd93c07bd6930970f0ff2055ee569914ad12ecf1de5e970c","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000ea3480\u003e: \n failed to create HCP cluster ea-cluster: failed starting cluster creation \"ea-cluster\" in resourcegroup=\"external-auth-cluster-pwsp6d\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-pwsp6d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster ea-cluster: failed starting cluster creation \\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-pwsp6d\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-pwsp6d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000ea3460\u003e{\n msg: \"failed starting cluster creation \\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-pwsp6d\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-pwsp6d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0011faa20\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"32\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"8a12884d-371f-4a60-a346-9bde4fef1bdc\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:14:58 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000ea33a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000ea3480\u003e: \n failed to create HCP cluster ea-cluster: failed starting cluster creation \"ea-cluster\" in resourcegroup=\"external-auth-cluster-pwsp6d\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-pwsp6d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster ea-cluster: failed starting cluster creation \\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-pwsp6d\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-pwsp6d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000ea3460\u003e{\n msg: \"failed starting cluster creation \\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-pwsp6d\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-pwsp6d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0011faa20\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"32\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"8a12884d-371f-4a60-a346-9bde4fef1bdc\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:14:58 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000ea33a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"401bf9e291581002825780df3fb4b7e53257787fe51de289cb4d70bdb3f7a9da","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Unsupported value: \"\": supported values: \"Private; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"d0a79b618ac6c3b247bb04063f3da6715a726b5902bbb07a0496f34818a88ee7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033384666504892416","occurred_at":"2026-03-16T03:27:37Z","failed_at":"e2e","signature_id":"99b55227703320ed5f7422f7292fbfc28ac17aa84433666f407175d665a9c9ae","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00060d400\u003e: \n failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-x86ztc\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-x86ztc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed starting cluster creation \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-x86ztc\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-x86ztc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00137d320\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"9bf7be7d-f58d-4898-9dec-9ecd01e3efa3\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:15:37 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"34\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00060d160\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 50, 32, 101, 114, 114, 111, 114, 115, 32, 111, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 100, 101, 116, 97, 105, 108, 115, 34, 58, 32, 91, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 82, 101, 113, 117, 105, 114, 101, 100, 32, 118, 97, 108, 117, 101, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 112, 114, 111, 112, 101, 114, 116, 105, 101, 115, 46, 101, 116, 99, 100, 46, 100, 97, 116, 97, 69, 110, 99, 114, 121, 112, 116, 105, 111, 110, 46, 99, 117, 115, 116, 111, 109, 101, 114, 77, 97, 110, 97, 103, 101, 100, 46, 107, 109, 115, 46, 118, 105, 115, 105, 98, 105, 108, 105, 116, 121, 34, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 125, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 4...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00060d400\u003e: \n failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-x86ztc\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-x86ztc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed starting cluster creation \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-x86ztc\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-x86ztc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00137d320\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"9bf7be7d-f58d-4898-9dec-9ecd01e3efa3\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:15:37 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"34\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00060d160\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 50, 32, 101, 114, 114, 111, 114, 115, 32, 111, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 100, 101, 116, 97, 105, 108, 115, 34, 58, 32, 91, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 82, 101, 113, 117, 105, 114, 101, 100, 32, 118, 97, 108, 117, 101, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 112, 114, 111, 112, 101, 114, 116, 105, 101, 115, 46, 101, 116, 99, 100, 46, 100, 97, 116, 97, 69, 110, 99, 114, 121, 112, 116, 105, 111, 110, 46, 99, 117, 115, 116, 111, 109, 101, 114, 77, 97, 110, 97, 103, 101, 100, 46, 107, 109, 115, 46, 118, 105, 115, 105, 98, 105, 108, 105, 116, 121, 34, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 125, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 4...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"401bf9e291581002825780df3fb4b7e53257787fe51de289cb4d70bdb3f7a9da","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Unsupported value: \"\": supported values: \"Private; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"d0fc6e84e80fc6cf32ce70f4bbca0f397aee190c121cad0875fce3cc0295272e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033384666504892416","occurred_at":"2026-03-16T03:27:37Z","failed_at":"e2e","signature_id":"c363d32b30daedd12046e216cbf440b1acc50a522761443a8f6bb1f0608f3708","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00074e520\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-cldhf9\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-cldhf9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-cldhf9\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-cldhf9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00074e440\u003e{\n msg: \"failed starting cluster creation \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-cldhf9\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-cldhf9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001131d10\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Request-Id\": [\n \"6cfc33f1-9c50-4fb2-9e3f-80eb42e82879\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:14:49 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"36\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000717f60\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00074e520\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-cldhf9\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-cldhf9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-cldhf9\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-cldhf9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00074e440\u003e{\n msg: \"failed starting cluster creation \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-cldhf9\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-cldhf9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001131d10\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Request-Id\": [\n \"6cfc33f1-9c50-4fb2-9e3f-80eb42e82879\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:14:49 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"36\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000717f60\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"401bf9e291581002825780df3fb4b7e53257787fe51de289cb4d70bdb3f7a9da","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Unsupported value: \"\": supported values: \"Private; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"d6f8b5a6087e662ce7231c5273e463dcbbcab909d9e305f780c4431abcbab02b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033384666504892416","occurred_at":"2026-03-16T03:27:37Z","failed_at":"e2e","signature_id":"051ec3efd25be81112c28a71153806f2226486cca606b5ebd7d316c71a7e1539","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00145d620\u003e: \n failed to create HCP cluster cluster-zstream-4-21-wvf2s7: failed starting cluster creation \"cluster-zstream-4-21-wvf2s7\" in resourcegroup=\"rg-zstream-upgrade-4-21-6cn5dr\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-6cn5dr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-wvf2s7\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-21-wvf2s7: failed starting cluster creation \\\"cluster-zstream-4-21-wvf2s7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-6cn5dr\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-6cn5dr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-wvf2s7\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00145d600\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-21-wvf2s7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-6cn5dr\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-6cn5dr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-wvf2s7\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012b43c0\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"2a932ba1-9dfa-4fda-a922-9a363b868873\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:17:56 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"30\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00145d540\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 9...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00145d620\u003e: \n failed to create HCP cluster cluster-zstream-4-21-wvf2s7: failed starting cluster creation \"cluster-zstream-4-21-wvf2s7\" in resourcegroup=\"rg-zstream-upgrade-4-21-6cn5dr\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-6cn5dr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-wvf2s7\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-21-wvf2s7: failed starting cluster creation \\\"cluster-zstream-4-21-wvf2s7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-6cn5dr\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-6cn5dr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-wvf2s7\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00145d600\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-21-wvf2s7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-6cn5dr\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-6cn5dr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-wvf2s7\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012b43c0\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"2a932ba1-9dfa-4fda-a922-9a363b868873\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:17:56 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"30\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00145d540\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 9...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"401bf9e291581002825780df3fb4b7e53257787fe51de289cb4d70bdb3f7a9da","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Unsupported value: \"\": supported values: \"Private; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"d7b368fa822e89bfd692af6d7893653f6842d19ba821957503caab346ff90d7d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033384666504892416","occurred_at":"2026-03-16T03:27:37Z","failed_at":"e2e","signature_id":"e5d7b1304d9a54137087d60bfce223b1d6b0393cb2a4138d8720f0d87639c750","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001560a60\u003e: \n failed to create HCP cluster node-limit-cluster: failed starting cluster creation \"node-limit-cluster\" in resourcegroup=\"node-limit-test-jzmjwm\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-jzmjwm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster node-limit-cluster: failed starting cluster creation \\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-jzmjwm\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-jzmjwm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001560a40\u003e{\n msg: \"failed starting cluster creation \\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-jzmjwm\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-jzmjwm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0011c5290\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"58c17626-9461-4842-b638-bc79c8bd43da\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:17:56 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"30\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc001560980\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 3...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001560a60\u003e: \n failed to create HCP cluster node-limit-cluster: failed starting cluster creation \"node-limit-cluster\" in resourcegroup=\"node-limit-test-jzmjwm\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-jzmjwm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster node-limit-cluster: failed starting cluster creation \\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-jzmjwm\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-jzmjwm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001560a40\u003e{\n msg: \"failed starting cluster creation \\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-jzmjwm\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-jzmjwm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0011c5290\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"58c17626-9461-4842-b638-bc79c8bd43da\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:17:56 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"30\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc001560980\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 3...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"401bf9e291581002825780df3fb4b7e53257787fe51de289cb4d70bdb3f7a9da","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Unsupported value: \"\": supported values: \"Private; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"dc254bd16ac2a8fd521cdf53ca3b372c9f182e3febdd1e07ef343ded4da486dd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033384666504892416","occurred_at":"2026-03-16T03:27:37Z","failed_at":"e2e","signature_id":"7da5f7e3a1c453e711567f74dc0fd7fa4acbafe19893fabfca9ee2ee9c94ee05","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0004934e0\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-w8rs7x\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-w8rs7x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-w8rs7x\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-w8rs7x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0004934c0\u003e{\n msg: \"failed starting cluster creation \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-w8rs7x\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-w8rs7x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000cfcd50\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Mon, 16 Mar 2026 04:14:49 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"34\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"69aaf16e-7144-4085-a11e-568c44599939\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000493220\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 3...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0004934e0\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-w8rs7x\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-w8rs7x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-w8rs7x\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-w8rs7x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0004934c0\u003e{\n msg: \"failed starting cluster creation \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-w8rs7x\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-w8rs7x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000cfcd50\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Mon, 16 Mar 2026 04:14:49 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"34\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"69aaf16e-7144-4085-a11e-568c44599939\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000493220\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 3...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"401bf9e291581002825780df3fb4b7e53257787fe51de289cb4d70bdb3f7a9da","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Unsupported value: \"\": supported values: \"Private; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"f31d8836d709f217b0b284a877d34f50ea0af1a82df45b2d7810858405c81732","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033384666504892416","occurred_at":"2026-03-16T03:27:37Z","failed_at":"e2e","signature_id":"0e59fbc7f56e6b0b5c3e268aa59bf826955ed47b0092f511ad27083ce247fd11","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b103a0\u003e: \n failed to create HCP cluster cluster-zstream-4-19-ccltmq: failed starting cluster creation \"cluster-zstream-4-19-ccltmq\" in resourcegroup=\"rg-zstream-upgrade-4-19-d5n6c6\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-d5n6c6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-ccltmq\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-ccltmq: failed starting cluster creation \\\"cluster-zstream-4-19-ccltmq\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-d5n6c6\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-d5n6c6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-ccltmq\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000b10380\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-ccltmq\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-d5n6c6\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-d5n6c6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-ccltmq\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000251980\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"55e0ab98-2366-4911-9f7b-f4c7195b7447\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:18:03 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"1035\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000b10040\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b103a0\u003e: \n failed to create HCP cluster cluster-zstream-4-19-ccltmq: failed starting cluster creation \"cluster-zstream-4-19-ccltmq\" in resourcegroup=\"rg-zstream-upgrade-4-19-d5n6c6\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-d5n6c6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-ccltmq\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Unsupported value: \\\"\\\": supported values: \\\"Private\\\", \\\"Public\\\"\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-ccltmq: failed starting cluster creation \\\"cluster-zstream-4-19-ccltmq\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-d5n6c6\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-d5n6c6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-ccltmq\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000b10380\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-ccltmq\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-d5n6c6\\\": PUT https://rp.j4892416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-d5n6c6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-ccltmq\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Unsupported value: \\\\\\\"\\\\\\\": supported values: \\\\\\\"Private\\\\\\\", \\\\\\\"Public\\\\\\\"\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000251980\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"55e0ab98-2366-4911-9f7b-f4c7195b7447\",\n ],\n \"Date\": [\n \"Mon, 16 Mar 2026 04:18:03 GMT\",\n ],\n \"Content-Length\": [\"614\"],\n \"X-Envoy-Upstream-Service-Time\": [\"1035\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000b10040\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"401bf9e291581002825780df3fb4b7e53257787fe51de289cb4d70bdb3f7a9da","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Unsupported value: \"\": supported values: \"Private; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}}],"failure_pattern_summary":{"signature_count":25,"failure_pattern_ids":["401bf9e291581002825780df3fb4b7e53257787fe51de289cb4d70bdb3f7a9da"],"matched_occurrences":25,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4475/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033381667984379904","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4475,"pr_state":"closed","pr_sha":"4d5b425cd52dc53a34dab2fe85f17202ddd9db29","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-16T03:15:42Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033358955824025600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4470,"pr_state":"open","pr_sha":"dfb641240c1f108be63315200630f51de45ba192","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-16T01:45:27Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033353043071733760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4473,"pr_state":"closed","pr_sha":"428d7803689b4cf003744ceed9f3f2e0fc9b7c02","final_merged_sha":"8b7576c81d01104dc238546ec2678952bf7afd61","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-16T01:21:57Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"8c7d3cd0b6ae7a3ea405d9ca7acc6eb0626498919345bf3f75fbff17d6063f47","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033353043071733760","occurred_at":"2026-03-16T01:21:57Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4449/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033345813895188480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4449,"pr_state":"closed","pr_sha":"54fcfdd15070338cbe7194c94e5fa8f655970db5","final_merged_sha":"205984e6ef0d1b76b1e624468408854ec2a65a8c","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-16T00:53:14Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2033341460748701696","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4471,"pr_state":"closed","pr_sha":"e4f48ae7498097713d8f691fbc75c82dbabe2343","final_merged_sha":"e4f48ae7498097713d8f691fbc75c82dbabe2343","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-16T00:35:56Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}