{"meta":{"date":"2026-03-19","timezone":"UTC","generated_at":"2026-10-31T23:15:17Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":97,"failed_runs":51,"runs_with_occurrences":51,"runs_with_failure_pattern_matches":44,"runs_with_unmatched_occurrences":7,"runs_without_occurrence_rows":46,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4545/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034781378851115008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4545,"pr_state":"closed","pr_sha":"7aba5e595948341263690a642b0eeb21b2c48bfc","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-19T23:57:39Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4548/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034775773692301312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4548,"pr_state":"closed","pr_sha":"17330f9f5e53a4a17db5a17fb8655104ae15312f","final_merged_sha":"17330f9f5e53a4a17db5a17fb8655104ae15312f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-19T23:35:23Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034773920745263104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4473,"pr_state":"closed","pr_sha":"a60ffd57d31ee93f8b2f07bffbed86eda10be999","final_merged_sha":"8b7576c81d01104dc238546ec2678952bf7afd61","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-19T23:28:01Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4536/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034770875445678080","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4536,"pr_state":"closed","pr_sha":"833532cff602417be25a9c830f42ff2d65952aab","final_merged_sha":"833532cff602417be25a9c830f42ff2d65952aab","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-19T23:15:55Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4535/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034766369068683264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4535,"pr_state":"closed","pr_sha":"3f902fae0079c098423ae1140a498f982a287e3f","final_merged_sha":"3f902fae0079c098423ae1140a498f982a287e3f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-19T22:57:59Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034761489453158400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4547,"pr_state":"closed","pr_sha":"ad55e38eaca04391113613f50f5d70c828340a36","final_merged_sha":"ad55e38eaca04391113613f50f5d70c828340a36","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-19T22:38:37Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034760866666123264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4457,"pr_state":"open","pr_sha":"d0996b7cdc84ad920ac1364b552090db3c739e42","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-19T22:36:09Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4513/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034751149755600896","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4513,"pr_state":"open","pr_sha":"dec6c48cb79a93cafd24130ab8af2c7306ad7a50","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-19T21:57:32Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034741024542167040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4305,"pr_state":"closed","pr_sha":"c0e24170903fb585e3d3c3f0c43e51336fc19704","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T21:17:18Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d8af94ecab540da791d778856d0c9ac2afac31699e641dcb2d132883f3b306d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034741024542167040","occurred_at":"2026-03-19T21:17:18Z","failed_at":"e2e","signature_id":"871f46647cc4875251f37fcac1fd5034bac43b46f03ff6a111f12d63981a0279","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:159]: Timed out after 60.001s.\nImageDigestMirrorSet CRDs should exist on the hosted cluster\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000e36110\u003e: \n IDMS verification failed:\n expected mirror source policy \"AllowContactingSource\" for source \"fake-source.example.com/fake\" (found: )\n {\n s: \"IDMS verification failed:\\nexpected mirror source policy \\\"AllowContactingSource\\\" for source \\\"fake-source.example.com/fake\\\" (found: )\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:159]: Timed out after 60.001s.\nImageDigestMirrorSet CRDs should exist on the hosted cluster\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000e36110\u003e: \n IDMS verification failed:\n expected mirror source policy \"AllowContactingSource\" for source \"fake-source.example.com/fake\" (found: )\n {\n s: \"IDMS verification failed:\\nexpected mirror source policy \\\"AllowContactingSource\\\" for source \\\"fake-source.example.com/fake\\\" (found: )\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0fe1b5bcc0409df5e5d428f89aaaeb6aba5c6cb04bb512cb5b7bf89e54afdbd","failure_pattern":"expected mirror source policy \"AllowContactingSource\" for source \"fake-source.example.com/fake\" (found: )","search_query":"expected mirror source policy \"AllowContactingSource\" for source \"fake-source.example.com/fake\" (found: )"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f0fe1b5bcc0409df5e5d428f89aaaeb6aba5c6cb04bb512cb5b7bf89e54afdbd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4484/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034740941859852288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4484,"pr_state":"closed","pr_sha":"ed1149a0319fe04fe728f3c36156a36617844244","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-19T21:16:58Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4425/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034740903842680832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4425,"pr_state":"closed","pr_sha":"855eea85f3052237fc8528c6d280c0ac41ee4307","final_merged_sha":"855eea85f3052237fc8528c6d280c0ac41ee4307","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-19T21:16:49Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4425/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034740251410305024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4425,"pr_state":"closed","pr_sha":"855eea85f3052237fc8528c6d280c0ac41ee4307","final_merged_sha":"855eea85f3052237fc8528c6d280c0ac41ee4307","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-19T21:14:11Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e99d0b7fd1f75028b90b120347744debba1bfa7f59890adeb47b5932df4965a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4425/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034740251410305024","occurred_at":"2026-03-19T21:14:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4234/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034739607819522048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4234,"pr_state":"closed","pr_sha":"ac3354ace12fb48d60cedc2365d42cdcd0e45759","final_merged_sha":"ac3354ace12fb48d60cedc2365d42cdcd0e45759","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-19T21:11:40Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3973/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034736730975768576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3973,"pr_state":"closed","pr_sha":"51eec28ae4c5247abd301426039daba35df0769f","final_merged_sha":"445d308e2fc4c51ba6b9b76df86f35addde8a91b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T21:00:14Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"091fc9398e5a023cbf462414852349d128c09a098be8b19b81a7d795c9717d0e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3973/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034736730975768576","occurred_at":"2026-03-19T21:00:14Z","failed_at":"e2e","signature_id":"db3344929edb8cf62ce9179325b2a9d761ec006e1dfef850bf44edcdb39d7d77","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000faad50\u003e: \n failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-fkdqkt, error: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-fkdqkt\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-fkdqkt, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-fkdqkt, error: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-fkdqkt\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-fkdqkt, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001781280\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-fkdqkt\",\n },\n \u003c*fmt.wrapErrors | 0xc000faad20\u003e{\n msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-fkdqkt\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-fkdqkt, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001781280\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-fkdqkt\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000faad50\u003e: \n failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-fkdqkt, error: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-fkdqkt\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-fkdqkt, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-fkdqkt, error: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-fkdqkt\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-fkdqkt, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001781280\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-fkdqkt\",\n },\n \u003c*fmt.wrapErrors | 0xc000faad20\u003e{\n msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-fkdqkt\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-fkdqkt, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001781280\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-fkdqkt\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-hv6bzb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-hv6bzb, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4537/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034731191302623232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4537,"pr_state":"closed","pr_sha":"71e53db487f9156c76b83e23dbed8f6c749c4bb1","final_merged_sha":"71e53db487f9156c76b83e23dbed8f6c749c4bb1","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-19T20:38:12Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4508/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034731137124798464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4508,"pr_state":"closed","pr_sha":"4da0d9f5994ef9590af727a3ca8f65f2ef8646b8","final_merged_sha":"4da0d9f5994ef9590af727a3ca8f65f2ef8646b8","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-19T20:37:52Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4425/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034712567217131520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4425,"pr_state":"closed","pr_sha":"855eea85f3052237fc8528c6d280c0ac41ee4307","final_merged_sha":"855eea85f3052237fc8528c6d280c0ac41ee4307","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-19T19:24:13Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"4e103c0bcffbcdd0f53e5082b8c2762abfa5f1943649747a0322ec99c8caa098","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4425/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034712567217131520","occurred_at":"2026-03-19T19:24:13Z","failed_at":"provision","signature_id":"913ffc41157493173a7d53891c8f1290dc0dc6ecc5aad4cb342fb0e9f67cf114","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Frontend/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-19T20:03:31.260Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-19T20:03:31.868Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy values=\"map[adminApi:map[namespace:aro-hcp-admin-api serviceAccount:admin-api] adminApiHost:admin.j7131520.hcpsvc.osadev.cloud apiBindPort:8443 audit:map[connectSocket:false] azure:map[clientId:9ae2dce2-36fd-4de1-9388-9b4bdd3466c8 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[databaseName:arohcpprow-rp-j7131520 databaseUrl:https://arohcpprow-rp-j7131520.documents.azure.com:443/ frontendMiClientId:3e51afb0-72c4-4214-8b19-e0e34a17f6b1 location:westus3] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:frontend-cert-prow-j7131520] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-l6h99n9b/pipeline@sha256:b84a27900c8a20ec255c2885caeb02d0d956d9ae792d8dcfba0223f93828f499 replicas:2 zoneCount:4] exitOnPanic:true mise:map[deploy:false policyLabel:ARM Policy] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-l6h99n9b/pipeline:pull workloadIdentityClientId:20c85050-271a-4f6d-b919-9e226971e33c workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:frontend workloadIdentityClientId:3e51afb0-72c4-4214-8b19-e0e34a17f6b1 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tracing:map[address:http://ingest.observability:4318 exporter:otlp] virtualService:map[host:rp.j7131520.hcpsvc.osadev.cloud]]\"\ntime=2026-03-19T20:03:31.869Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy\ntime=2026-03-19T20:03:32.044Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-03-19T20:03:32.363Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy dryRun=false\ntime=2026-03-19T20:03:32.363Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy\ntime=2026-03-19T20:03:32.472Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy\ntime=2026-03-19T20:08:38.325Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-frontend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-19T20:08:38.325Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy\ntime=2026-03-19T20:08:38.325Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy\ntime=2026-03-19T20:08:41.331Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy release=aro-hcp-frontend-dev namespace=aro-hcp status=failed description=\"Release \\\"aro-hcp-frontend-dev\\\" failed: resource not ready, name: aro-hcp-frontend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[adminApi:map[namespace:aro-hcp-admin-api serviceAccount:admin-api] adminApiHost:admin.j7131520.hcpsvc.osadev.cloud apiBindPort:8443 audit:map[connectSocket:false] azure:map[clientId:9ae2dce2-36fd-4de1-9388-9b4bdd3466c8 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[databaseName:arohcpprow-rp-j7131520 databaseUrl:https://arohcpprow-rp-j7131520.documents.azure.com:443/ frontendMiClientId:3e51afb0-72c4-4214-8b19-e0e34a17f6b1 location:westus3] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:frontend-cert-prow-j7131520] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-l6h99n9b/pipeline@sha256:b84a27900c8a20ec255c2885caeb02d0d956d9ae792d8dcfba0223f93828f499 replicas:2 zoneCount:4] exitOnPanic:true mise:map[deploy:false policyLabel:ARM Policy] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-l6h99n9b/pipeline:pull workloadIdentityClientId:20c85050-271a-4f6d-b919-9e226971e33c workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:frontend workloadIdentityClientId:3e51afb0-72c4-4214-8b19-e0e34a17f6b1 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tracing:map[address:http://ingest.observability:4318 exporter:otlp] virtualService:map[host:rp.j7131520.hcpsvc.osadev.cloud]]\"\ntime=2026-03-19T20:08:41.333Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5xUzW7bMAy++ykEnxrAKrIEGLoMPfRn2GHoEKDDDh12kCXGYSuTBkUny9CHH5ykjVd4Qbvrp++HpCRmEdQIJG7FQzLnJjh16soIJ1zeg9cvSGGWVJCqwuygr66GZwhWQFskNc4/waMfmTHG5FdMC6xuXJMXJl8IkwIF67doBzlhu/RNXuzotyAr9NA7sU+it7FtDSro04DqotUlC/52ikxzjug3W1aMvLYu1EjWNWiVj0UfNdln/7eeWJdIQ/M5KsOkyBapEkhDjc8BpDMAUvRbhzdO7ha8gM6FVxhArqLbxTwrk99es3tIL2vZF8DhGpO0TZd92YYK9G03feE9t6T90AHyNTSRNzXQa93/FhxeKYjiohsVWIGFQFqCHG3wwsu8jfESKezv79msaWO05eHgRQmfncLabfoFVzvIwi8FIRePJu8HdMOEyvLKvr+jaOviwDdaDT/e7OfH7KEtQQgU0qfu46fs0ayXIGAUa0jq6saUoGsAMifBKXTwST4ZT97b8dS++/BtMp6Np7Pp5C4fmdNT80/O2Wx6dpePRtmjuWck84AUzpEIpLeumMxhS/XX08u9lD2aRrg7PRRamIbDjvw6k8JErgpTQ0qu6ixZAogpN73mAyT/JwAA//9EUhfPVAUAAA==\"\ntime=2026-03-19T20:08:41.335Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy\ntime=2026-03-19T20:08:41.336Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-frontend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"42454831adde3c75179282c1ce6d94be508cc22e8880a9bd19abc0e54e5a4d48","failure_pattern":"resource not ready, name: aro-hcp-frontend, kind: Deployment, status: InProgress context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-frontend, kind: Deployment, status: InProgress\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["42454831adde3c75179282c1ce6d94be508cc22e8880a9bd19abc0e54e5a4d48"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034709608861274112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4004,"pr_state":"closed","pr_sha":"1d29b3510e9e18022079152f3b5eaeb20910a353","final_merged_sha":"9feee09ab6996cfe7dd433b262345799b7851593","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-19T19:12:28Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034708388679520256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4508,"pr_state":"closed","pr_sha":"a2ca195603032cc033a8e2ebe803918b756d5580","final_merged_sha":"4da0d9f5994ef9590af727a3ca8f65f2ef8646b8","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T19:07:35Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"9d831eb01755322de49f403f214ed4c66d9f78989c712808dc4f800188155093","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034708388679520256","occurred_at":"2026-03-19T19:07:35Z","failed_at":"e2e","signature_id":"1be72eaa445c606f57b34a1ff6282ab757493435895abeb3803f31e0f7ca7747","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:284]: Unexpected error:\n \u003c*errors.StatusError | 0xc000e04960\u003e: \n pods \"wi-test-pod\" is forbidden: error fetching namespace \"e2e-oidc-wi\": unable to find annotation openshift.io/sa.scc.uid-range\n {\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"pods \\\"wi-test-pod\\\" is forbidden: error fetching namespace \\\"e2e-oidc-wi\\\": unable to find annotation openshift.io/sa.scc.uid-range\",\n Reason: \"Forbidden\",\n Details: {Name: \"wi-test-pod\", Group: \"\", Kind: \"pods\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:284]: Unexpected error:\n \u003c*errors.StatusError | 0xc000e04960\u003e: \n pods \"wi-test-pod\" is forbidden: error fetching namespace \"e2e-oidc-wi\": unable to find annotation openshift.io/sa.scc.uid-range\n {\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"pods \\\"wi-test-pod\\\" is forbidden: error fetching namespace \\\"e2e-oidc-wi\\\": unable to find annotation openshift.io/sa.scc.uid-range\",\n Reason: \"Forbidden\",\n Details: {Name: \"wi-test-pod\", Group: \"\", Kind: \"pods\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"31bab740c670d74d59ae23b10d74f51b9de483710d0d1e65e2cdb24e5ecc9063","failure_pattern":"pod \"\u003cpod\u003e\" is forbidden: error fetching namespace \"\u003cnamespace\u003e\": unable to find annotation openshift.io/sa.scc.uid-range","search_query":"pods \"wi-test-pod\" is forbidden: error fetching namespace \"e2e-oidc-wi\": unable to find annotation openshift.io/sa.scc.uid-range"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["31bab740c670d74d59ae23b10d74f51b9de483710d0d1e65e2cdb24e5ecc9063"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4545/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034703138790838272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4545,"pr_state":"closed","pr_sha":"ed91e426dd0b93d771d003594d7e26f992d63d40","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T18:46:45Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"22c70c020a87867000e35df40629f3542961f1bd1afe064a612316e12929a2c6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4545/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034703138790838272","occurred_at":"2026-03-19T18:46:45Z","failed_at":"provision","signature_id":"14e1139051b92a0259280ab4bba3ccc679edd3541b36e2204a8d4ec84f155427","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/prometheus","test_suite":"step graph","failure_text":"time=2026-03-19T19:25:28.273Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-03-19T19:25:28.988Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:d7e8257f8d8bce64b6df55f81fba92011a6a77269b3350f8b997b152af348dba]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:60cce7ea-f8ae-47ea-a2d1-1660743c2fc5]] prometheusSpec:map[externalLabels:map[cluster:prow-j0838272-svc] hcpRemoteWriteUrl:NONE image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:61c6060c6bd283bf8633baf18e999c74ddb2cd65004784eb6a70204c6afa0e1f] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j0838272-svc-dcdl.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-5a84791bf5eb40b5a04b1a814789adaf/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-19T19:25:28.988Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-19T19:25:29.455Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus namespace=prometheus\ntime=2026-03-19T19:25:29.870Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus dryRun=false\ntime=2026-03-19T19:25:29.870Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-19T19:25:29.984Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-19T19:31:26.929Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"resource not ready, name: prometheus-operator, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-19T19:31:26.929Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-19T19:31:26.929Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-19T19:31:34.458Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus release=arohcp-monitor namespace=prometheus status=failed description=\"Release \\\"arohcp-monitor\\\" failed: resource not ready, name: prometheus-operator, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:d7e8257f8d8bce64b6df55f81fba92011a6a77269b3350f8b997b152af348dba]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:60cce7ea-f8ae-47ea-a2d1-1660743c2fc5]] prometheusSpec:map[externalLabels:map[cluster:prow-j0838272-svc] hcpRemoteWriteUrl:NONE image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:61c6060c6bd283bf8633baf18e999c74ddb2cd65004784eb6a70204c6afa0e1f] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j0838272-svc-dcdl.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-5a84791bf5eb40b5a04b1a814789adaf/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-19T19:31:34.460Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/6yXz2/bOgzH7/4rhJwaIAraFK94zUMP773tNHQoumEDOuzASJyjxhIFim4WIH/84Lhp3B9pXTtX+suPKZGiqKxAUYyJSjaY1IWyICAwK/CIZrdo5JMLdpqEXchHqjZ9Bo8PJrzDsLGkCGZrHv7IlFJq8AX5zhn81xgqgwxGagBMcxO1p+CEWC/KGeokIKg9CjuTKlFk8ihzLNNgtA+002iKyCDE73d92eN/Cr9cfglxE7CHbWg6oYgLedJmo/C1ol7DKgn6B0JRJkG+pgJbL3qP7551vqluiu63oP/+v2vjP2AsaOWxV+IfQ1r/+7IUqFL1HWdzokWd0ZJBHIUnILDepXRv3/p/g8LZXoRGYv5zwbqQ96mEBqJ9Qbzo1NRekb2sI2lzLK72fn9dfV0+q2QokMVDgBx5zGWBXSD1IdSMBYFF7oLIMSBD0TmExd+p9h0bCgIuIGsTS10myFEnNBRs0kICxYHoHj3xShsw82cHvh+S06F24B6YlnV/PBxxSbxwIa/6sJ6t5GAp295/vXCRrKZlwD0d6XVK1QcOWZdPeIEsakZDXDWEftgWTftNCAcUTBpi7El4mFz6YZIQQ94p/w3IZgTQEF1CvutcB49plaXAZ2NLF1YHxqZu8Hcklh6N+hGlq39AqY5/R/fOsbeeOFpSXh2vGpdxvzntxVt93wKyn/9ku2L5WM3zKVur5RwZlTiPScBHNUNZIgZ1ZEGwMh8NJseTM318qk/Ov56cTyd/TSfnN4OhGo/VPs3pyXRydjMYDrO1uiUX1MIFe+FCQG68Qiio3eOj+ep4+tzI1ioyVV93gY5UJFuL20FGqqB8pDym6tbO1orYIqvZqrF4i8n8CQAA//8FiIteKw0AAA==\"\ntime=2026-03-19T19:31:34.461Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-03-19T19:31:34.461Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: prometheus-operator, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d22bf2530e6de03ff6d365da9c58f32e0973da02f146398a601d1f0435f1751a","failure_pattern":"resource not ready, name: prometheus-operator, kind: Deployment, status: InProgress context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: prometheus-operator, kind: Deployment, status: InProgress\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d22bf2530e6de03ff6d365da9c58f32e0973da02f146398a601d1f0435f1751a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4544/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034702410932293632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4544,"pr_state":"open","pr_sha":"1a5b93428450e3b4897c3cd769fc9e0f954b60cf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T18:43:52Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"8bf74b05fb5e79b4a5a4a971fad8e04e6f98bcfe61459398405b610f35bb653b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4544/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034702410932293632","occurred_at":"2026-03-19T18:43:52Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4543/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034698452704694272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4543,"pr_state":"closed","pr_sha":"fe8a36903d2740bd51053d3ff294d917b4a8a045","final_merged_sha":"89c0aa6e9b5745eff17e497b7e08fde82fdde6b5","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T18:28:06Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3917202e3180e8bea14a7cdca1e927bd03fb04336341210818a26a4d0807ed83","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4543/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034698452704694272","occurred_at":"2026-03-19T18:28:06Z","failed_at":"provision","signature_id":"c027c4d12f6a3d02006d0282f3d5b1020328ed77f7bf43d76c7a5e7566e656ac","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-03-19T18:52:45.735Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-03-19T18:52:47.817Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-03-19T18:52:49.540Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=2504b525a81d38e7574160e2f0de8fb8c51a787cf186454a081120b8755f9cdb\ntime=2026-03-19T18:56:50.548Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j4694272-svc/providers/Microsoft.Resources/deployments/2504b525a81d38e7574160e2f0de8fb8c51a787cf186454a081120b8755f9cdb/operationStatuses/08584276605168554876\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4694272-svc/providers/Microsoft.Resources/deployments/cluster-nkopotfn76bws\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4694272/providers/Microsoft.Resources/deployments/oidc-storage\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4694272/providers/Microsoft.Resources/deployments/storage\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4694272/providers/Microsoft.Resources/deployments/storage\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4694272/providers/Microsoft.Resources/deployments/oidcStorageRbac\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4694272/providers/Microsoft.Resources/deployments/oidcStorageRbac\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4234/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034698192720760832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4234,"pr_state":"closed","pr_sha":"292d211d91532b8717e75977dc02a37771b0848e","final_merged_sha":"ac3354ace12fb48d60cedc2365d42cdcd0e45759","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T18:27:06Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"a512e8a258dc237aa95b949cdecdd9f559007f6f67ab4fa147a7abfc6af15b29","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4234/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034698192720760832","occurred_at":"2026-03-19T18:27:06Z","failed_at":"provision","signature_id":"c027c4d12f6a3d02006d0282f3d5b1020328ed77f7bf43d76c7a5e7566e656ac","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-03-19T18:52:31.438Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-03-19T18:52:33.182Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-03-19T18:52:35.043Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=3cc66decdbed55c45fdd4ee75868913bdbaa393a1029e6c0a3a6804a158f315f\ntime=2026-03-19T18:56:36.392Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j0760832-svc/providers/Microsoft.Resources/deployments/3cc66decdbed55c45fdd4ee75868913bdbaa393a1029e6c0a3a6804a158f315f/operationStatuses/08584276605314679492\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0760832-svc/providers/Microsoft.Resources/deployments/cluster-ty5h7c4eu4lcw\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4536/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034697480548913152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4536,"pr_state":"closed","pr_sha":"833532cff602417be25a9c830f42ff2d65952aab","final_merged_sha":"833532cff602417be25a9c830f42ff2d65952aab","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-19T18:24:16Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"264102561e7a97716cfd9139f5217e1c776b6b778cb96e59b1d24c676ad9cadb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4536/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034697480548913152","occurred_at":"2026-03-19T18:24:16Z","failed_at":"provision","signature_id":"0cfdbc69e36cd9409140a1d94f3eca54c19e2976278c5f5c45b1651fb5c59fcf","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster","test_suite":"step graph","failure_text":"time=2026-03-19T18:54:20.685Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-03-19T18:54:21.984Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster\ntime=2026-03-19T18:54:23.201Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster deployment=0bfdc9c537f9456f74c5b80bcd861a7f9d2b569219c8aa1b927dbc89314df45f\ntime=2026-03-19T18:57:54.234Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j8913152-mgmt-1/providers/Microsoft.Resources/deployments/0bfdc9c537f9456f74c5b80bcd861a7f9d2b569219c8aa1b927dbc89314df45f/operationStatuses/08584276604230221010\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8913152-mgmt-1/providers/Microsoft.Resources/deployments/cluster-w2b4uene52bv2\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8913152-mgmt-1/providers/Microsoft.Resources/deployments/prow-j8913152-mgmt-1-outbound-ip\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8913152-mgmt-1/providers/Microsoft.Resources/deployments/prow-j8913152-mgmt-1-outbound-ip\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034696887444967424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4004,"pr_state":"closed","pr_sha":"1d29b3510e9e18022079152f3b5eaeb20910a353","final_merged_sha":"9feee09ab6996cfe7dd433b262345799b7851593","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T18:21:55Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"59964af6ed655cea4248540b6e475688ffa4b3cfbc1389cf67b7e5c7764bc527","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034696887444967424","occurred_at":"2026-03-19T18:21:55Z","failed_at":"provision","signature_id":"c027c4d12f6a3d02006d0282f3d5b1020328ed77f7bf43d76c7a5e7566e656ac","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-03-19T18:46:17.431Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-03-19T18:46:20.647Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-03-19T18:46:22.202Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=80bb9318950571ddc624dbabe49f514c6d8f9c09f985a97c625bf27331d9acc0\ntime=2026-03-19T18:57:54.937Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j4967424-svc/providers/Microsoft.Resources/deployments/80bb9318950571ddc624dbabe49f514c6d8f9c09f985a97c625bf27331d9acc0/operationStatuses/08584276609041060102\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4967424-svc/providers/Microsoft.Resources/deployments/cluster-sj245grfkcdbe\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/096555cd-32ef-5afa-acc2-c55ce48633c7\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/096555cd-32ef-5afa-acc2-c55ce48633c7\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/ad556013-895e-5a6b-820b-5cf5e4c3463b\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/ad556013-895e-5a6b-820b-5cf5e4c3463b\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4513/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034692015664402432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4513,"pr_state":"open","pr_sha":"eee29a6733cdc1078caa24a72e1c288f5ff622dd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-19T18:02:24Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034690792215285760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4457,"pr_state":"open","pr_sha":"ec87a0b9337e82f98b09a03dd3584d7dfdbfd780","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T17:57:42Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"777a04d87b02940b49d03c236f0e6cbf8e4664577e974a6a1c82408db0ca71da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034690792215285760","occurred_at":"2026-03-19T17:57:42Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034689380060237824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4133,"pr_state":"closed","pr_sha":"fb6029b0e8644a9763ccd8d27b820c45b9a3b9af","final_merged_sha":"4f10ef55a408ea39a4f644c08d4ab2d01cf39c2a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T17:52:05Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"1b423f823546a717201e85b31c5b6dc2a17f661caea6f8dc38cf563246880883","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034689380060237824","occurred_at":"2026-03-19T17:52:05Z","failed_at":"e2e","signature_id":"683eef8a0e12932c0dd4763b812fe52b1043b9ded109cf178071653c635bccde","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to 500 replicas","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:156]: Expected success, but got an error:\n \u003c*errors.errorString | 0xc000113e10\u003e: \n expected 1 nodes, found 4\n {\n s: \"expected 1 nodes, found 4\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:156]: Expected success, but got an error:\n \u003c*errors.errorString | 0xc000113e10\u003e: \n expected 1 nodes, found 4\n {\n s: \"expected 1 nodes, found 4\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9a89ea5329f68616b6d025b109dbd853424e587dde1af33e0a8da5edd81ba9e1","failure_pattern":"expected 1 nodes, found 4","search_query":"expected 1 nodes, found 4"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9a89ea5329f68616b6d025b109dbd853424e587dde1af33e0a8da5edd81ba9e1"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034683213669994496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4508,"pr_state":"closed","pr_sha":"a2ca195603032cc033a8e2ebe803918b756d5580","final_merged_sha":"4da0d9f5994ef9590af727a3ca8f65f2ef8646b8","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-19T17:27:35Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034679798927724544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4305,"pr_state":"closed","pr_sha":"17b6a5a4eba730575119de10b4aa84b45c673819","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T17:14:01Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"40f7a4afc90ce2264230726f7efa8054d7f702ff07cbbfce93ca1020a8998b9b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034679798927724544","occurred_at":"2026-03-19T17:14:01Z","failed_at":"e2e","signature_id":"7e42f5bbd0be6fe7e7d94b37fbc8c94eb1c9f8b454298b479289844439e682af","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129]: Unexpected error:\n \u003c*exported.ResponseError | 0xc000558990\u003e: \n GET https://rp.j7724544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6e4be108-3630-4cd1-9bc4-5f9bf6c1d133\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6e4be108-3630-4cd1-9bc4-5f9bf6c1d133\",\n \"name\": \"6e4be108-3630-4cd1-9bc4-5f9bf6c1d133\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-19T17:55:14.004165795Z\",\n \"endTime\": \"2026-03-19T18:00:44.464234913Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-k9pmhk/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Thu, 19 Mar 2026 18:00:47 GMT\",\n ],\n \"Content-Length\": [\"1395\"],\n \"X-Envoy-Upstream-Service-Time\": [\"11\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"c0c30370-9a3a-4374-bfd7-67496bcec6a6\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0007801c0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 54, 101, 52, 98, 101, 49, 48, 56, 45, 51, 54, 51, 48, 45, 52, 99, 100, 49, 45, 57, 98, 99, 52, 45, 53, 102, 57, 98, 102, 54, 99, 49, 100, 49, 51, 51, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 54, 101, 52, 98, 101, 49, 48, 56, 45, 51, 54, 51, 48, 45, 52, 99, 100, 49, 45, 57, 98, 99, 52, 45, 53, 102, 57, 98, 102, 54, 99, 49, 100, 49, 51, 51, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 116, 117, 115, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 114, 116, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 57, 84, 49, 55, 58, 53, 53, 58, 49, 52, 46, 48, 48, 52, 49, 54, 53, 55, 57, 53, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 110, 100, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 57, 84, 49, 56, 58, 48, 48, 58, 52, 52, 46, 52, 54, 52, 50, 51, 52, 57, 49, 51, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 102, 97, 105, 108, 101, 100, 32, 116, 111, 32, 103, 101, 116, 32, 110, 101, 116, 119, 111, 114, 107, 32, 115, 101, 99, 117, 114, 105, 116, 121, 32, 103, 114, 111, 117, 112, 32, 39, 99, 117, 115, 116, 111, 109, 101, 114, 45, 110, 115, 103, 45, 110, 97, 109, 101, 39, 32, 100, 101, 116, 97, 105, 108, 115, 58, 32, 71, 69, 84, 32, 104, 116, 116, 112, 115, 58, 47, 47, 109, 97, 110, 97, 103, 101, 109, 101, 110, 116, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 99, 117, 115, 116, 111, 109, 101, 114, 45, 114, 103, 45, 107, 57, 112, 109, 104, 107, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 78, 101, 116, 119, 111, 114, 107, 47, 110, 101, 116, 119, 111, 114, 107, 83, 101, 99, 117, 114, 105, 116, 121, 71, 114, 111, 117, 112, 115, 47, 99, 117, 115, 116, 111, 109, 101, 114, 45, 110, 115, 103, 45, 110, 97, 109, 101, 92, 110, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129]: Unexpected error:\n \u003c*exported.ResponseError | 0xc000558990\u003e: \n GET https://rp.j7724544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6e4be108-3630-4cd1-9bc4-5f9bf6c1d133\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6e4be108-3630-4cd1-9bc4-5f9bf6c1d133\",\n \"name\": \"6e4be108-3630-4cd1-9bc4-5f9bf6c1d133\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-19T17:55:14.004165795Z\",\n \"endTime\": \"2026-03-19T18:00:44.464234913Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-k9pmhk/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Thu, 19 Mar 2026 18:00:47 GMT\",\n ],\n \"Content-Length\": [\"1395\"],\n \"X-Envoy-Upstream-Service-Time\": [\"11\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"c0c30370-9a3a-4374-bfd7-67496bcec6a6\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0007801c0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 54, 101, 52, 98, 101, 49, 48, 56, 45, 51, 54, 51, 48, 45, 52, 99, 100, 49, 45, 57, 98, 99, 52, 45, 53, 102, 57, 98, 102, 54, 99, 49, 100, 49, 51, 51, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 54, 101, 52, 98, 101, 49, 48, 56, 45, 51, 54, 51, 48, 45, 52, 99, 100, 49, 45, 57, 98, 99, 52, 45, 53, 102, 57, 98, 102, 54, 99, 49, 100, 49, 51, 51, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 116, 117, 115, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 114, 116, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 57, 84, 49, 55, 58, 53, 53, 58, 49, 52, 46, 48, 48, 52, 49, 54, 53, 55, 57, 53, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 110, 100, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 57, 84, 49, 56, 58, 48, 48, 58, 52, 52, 46, 52, 54, 52, 50, 51, 52, 57, 49, 51, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 102, 97, 105, 108, 101, 100, 32, 116, 111, 32, 103, 101, 116, 32, 110, 101, 116, 119, 111, 114, 107, 32, 115, 101, 99, 117, 114, 105, 116, 121, 32, 103, 114, 111, 117, 112, 32, 39, 99, 117, 115, 116, 111, 109, 101, 114, 45, 110, 115, 103, 45, 110, 97, 109, 101, 39, 32, 100, 101, 116, 97, 105, 108, 115, 58, 32, 71, 69, 84, 32, 104, 116, 116, 112, 115, 58, 47, 47, 109, 97, 110, 97, 103, 101, 109, 101, 110, 116, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 99, 117, 115, 116, 111, 109, 101, 114, 45, 114, 103, 45, 107, 57, 112, 109, 104, 107, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 78, 101, 116, 119, 111, 114, 107, 47, 110, 101, 116, 119, 111, 114, 107, 83, 101, 99, 117, 114, 105, 116, 121, 71, 114, 111, 117, 112, 115, 47, 99, 117, 115, 116, 111, 109, 101, 114, 45, 110, 115, 103, 45, 110, 97, 109, 101, 92, 110, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c99cd07f1c6a61d42801878118c1713569775a2f72ee3dc4bc6292feb92daa9a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034679798927724544","occurred_at":"2026-03-19T17:14:01Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034674971761971200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4351,"pr_state":"closed","pr_sha":"131806c6037c274a80c20df0a02233247a78c57e","final_merged_sha":"131806c6037c274a80c20df0a02233247a78c57e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-19T16:54:50Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034674233283448832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4351,"pr_state":"closed","pr_sha":"131806c6037c274a80c20df0a02233247a78c57e","final_merged_sha":"131806c6037c274a80c20df0a02233247a78c57e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-19T16:51:54Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034673450164948992","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4351,"pr_state":"closed","pr_sha":"131806c6037c274a80c20df0a02233247a78c57e","final_merged_sha":"131806c6037c274a80c20df0a02233247a78c57e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-19T16:48:47Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"6872cd4f55ec192aa95cff0749c2fa344afac169c7f3d9883b47a9465aedd0e5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034673450164948992","occurred_at":"2026-03-19T16:48:47Z","failed_at":"e2e","signature_id":"968c4404e252d3d22b014956f23d13387b9f325d10e1227dc6d622fe943f2bf1","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0013a82a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-xk2ks9\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000929200\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-xk2ks9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009291e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-xk2ks9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009291c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-xk2ks9\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09c0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0013a82a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-xk2ks9\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000929200\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-xk2ks9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009291e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-xk2ks9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009291c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-xk2ks9\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09c0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-xk2ks9\" to finish deleting: context canceled"}},{"row_id":"8484f80dbb7d8597ab7536ca18e7853fd1a46836f529da9cd0470186993bc805","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034673450164948992","occurred_at":"2026-03-19T16:48:47Z","failed_at":"e2e","signature_id":"cb971456b4cafbb113df2140f2397f8f709b26865ee870367aa28333c9b9ebee","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00168a138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-l9pvkm\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000c09040\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-l9pvkm\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000c09020\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-l9pvkm\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000c09000\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-l9pvkm\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09c0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00168a138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-l9pvkm\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000c09040\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-l9pvkm\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000c09020\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-l9pvkm\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000c09000\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-l9pvkm\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09c0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-xk2ks9\" to finish deleting: context canceled"}},{"row_id":"9ef41c81f170523f1366dedcbdd7c1fdcbb719d1779e8c33b1033c950eaab365","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034673450164948992","occurred_at":"2026-03-19T16:48:47Z","failed_at":"e2e","signature_id":"db242b6b8c7d7f8958638c3063a698e3ce2ffde3b18282c0b710679211865b2f","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000eaced0\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-tmg7sm\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-tmg7sm, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-tmg7sm\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-tmg7sm, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00066b770\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-tmg7sm\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000eaced0\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-tmg7sm\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-tmg7sm, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-tmg7sm\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-tmg7sm, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00066b770\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-tmg7sm\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-tmg7sm\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-tmg7sm, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4513/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034671383274852352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4513,"pr_state":"open","pr_sha":"9c07c65c12691f823e1cfa3651d2faa30d6579ef","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T16:40:32Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"fe4ab20f92306a0eb555e02ee1c9cb7d0b892eec1434c43a4c908e8ecd7cda14","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4513/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034671383274852352","occurred_at":"2026-03-19T16:40:32Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034671022241746944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4505,"pr_state":"closed","pr_sha":"9e4e5bd89728a904bc37ae0fc5c1633e841f6791","final_merged_sha":"ec8c17ac019d36b2be42061c5ce822f44607110b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T16:39:08Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"2cbfae36b30591f7333dc8d568910c679eb1de1d0b554c1a7c2d39aac15bfb80","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034671022241746944","occurred_at":"2026-03-19T16:39:08Z","failed_at":"e2e","signature_id":"971985e409c2ca5ce6603767ab9ba01b1c3e2c1dd7c83e8baa4e078bec70cba9","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000792440\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-vrwqp2\" to finish creating: GET https://rp.j1746944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ef11d5a-18cb-48ed-afc1-00eea0d23317\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ef11d5a-18cb-48ed-afc1-00eea0d23317\",\n \"name\": \"1ef11d5a-18cb-48ed-afc1-00eea0d23317\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-19T17:47:14.755191638Z\",\n \"endTime\": \"2026-03-19T17:49:13.219999333Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-vrwqp2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\\\"cluster-np-labels-taints\\\" in resourcegroup=\\\"rg-np-labels-taints-vrwqp2\\\" to finish creating: GET https://rp.j1746944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ef11d5a-18cb-48ed-afc1-00eea0d23317\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ef11d5a-18cb-48ed-afc1-00eea0d23317\\\",\\n \\\"name\\\": \\\"1ef11d5a-18cb-48ed-afc1-00eea0d23317\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-19T17:47:14.755191638Z\\\",\\n \\\"endTime\\\": \\\"2026-03-19T17:49:13.219999333Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-vrwqp2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000792420\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-np-labels-taints\\\" in resourcegroup=\\\"rg-np-labels-taints-vrwqp2\\\" to finish creating: GET https://rp.j1746944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ef11d5a-18cb-48ed-afc1-00eea0d23317\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ef11d5a-18cb-48ed-afc1-00eea0d23317\\\",\\n \\\"name\\\": \\\"1ef11d5a-18cb-48ed-afc1-00eea0d23317\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-19T17:47:14.755191638Z\\\",\\n \\\"endTime\\\": \\\"2026-03-19T17:49:13.219999333Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-vrwqp2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again afte...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000792440\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-vrwqp2\" to finish creating: GET https://rp.j1746944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ef11d5a-18cb-48ed-afc1-00eea0d23317\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ef11d5a-18cb-48ed-afc1-00eea0d23317\",\n \"name\": \"1ef11d5a-18cb-48ed-afc1-00eea0d23317\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-19T17:47:14.755191638Z\",\n \"endTime\": \"2026-03-19T17:49:13.219999333Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-vrwqp2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\\\"cluster-np-labels-taints\\\" in resourcegroup=\\\"rg-np-labels-taints-vrwqp2\\\" to finish creating: GET https://rp.j1746944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ef11d5a-18cb-48ed-afc1-00eea0d23317\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ef11d5a-18cb-48ed-afc1-00eea0d23317\\\",\\n \\\"name\\\": \\\"1ef11d5a-18cb-48ed-afc1-00eea0d23317\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-19T17:47:14.755191638Z\\\",\\n \\\"endTime\\\": \\\"2026-03-19T17:49:13.219999333Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-vrwqp2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000792420\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-np-labels-taints\\\" in resourcegroup=\\\"rg-np-labels-taints-vrwqp2\\\" to finish creating: GET https://rp.j1746944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ef11d5a-18cb-48ed-afc1-00eea0d23317\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ef11d5a-18cb-48ed-afc1-00eea0d23317\\\",\\n \\\"name\\\": \\\"1ef11d5a-18cb-48ed-afc1-00eea0d23317\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-19T17:47:14.755191638Z\\\",\\n \\\"endTime\\\": \\\"2026-03-19T17:49:13.219999333Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-vrwqp2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again afte...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"552e836fd98e6d4fc3076c2bb5853061a171bbee0042c21805abac67602887fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034671022241746944","occurred_at":"2026-03-19T16:39:08Z","failed_at":"e2e","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","test_name":"Customer should upgrade a nodepool from 4.20 to 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_minor_upgrade.go:185]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_minor_upgrade.go:185]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"e599ba98c1988495ed433f05f6b919e897b0fe4cbfbdd08a9919790198731009","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034671022241746944","occurred_at":"2026-03-19T16:39:08Z","failed_at":"e2e","signature_id":"4df69cd4467db6f81900869e720d2f419e831519c2c6858a01c5c780a6343416","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000f663a0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-bq8pz8\" to finish creating: GET https://rp.j1746944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09c2c3d3-9afa-4303-846a-0e7c745a8cbe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09c2c3d3-9afa-4303-846a-0e7c745a8cbe\",\n \"name\": \"09c2c3d3-9afa-4303-846a-0e7c745a8cbe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-19T17:47:14.335034804Z\",\n \"endTime\": \"2026-03-19T17:49:12.841643046Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Failed to get virtual network 'customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-bq8pz8/providers/Microsoft.Network/virtualNetworks/customer-vnet\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-bq8pz8\\\" to finish creating: GET https://rp.j1746944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09c2c3d3-9afa-4303-846a-0e7c745a8cbe\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09c2c3d3-9afa-4303-846a-0e7c745a8cbe\\\",\\n \\\"name\\\": \\\"09c2c3d3-9afa-4303-846a-0e7c745a8cbe\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-19T17:47:14.335034804Z\\\",\\n \\\"endTime\\\": \\\"2026-03-19T17:49:12.841643046Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-bq8pz8/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000f66380\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-bq8pz8\\\" to finish creating: GET https://rp.j1746944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09c2c3d3-9afa-4303-846a-0e7c745a8cbe\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09c2c3d3-9afa-4303-846a-0e7c745a8cbe\\\",\\n \\\"name\\\": \\\"09c2c3d3-9afa-4303-846a-0e7c745a8cbe\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-19T17:47:14.335034804Z\\\",\\n \\\"endTime\\\": \\\"2026-03-19T17:49:12.841643046Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-bq8pz8/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.m...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000f663a0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-bq8pz8\" to finish creating: GET https://rp.j1746944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09c2c3d3-9afa-4303-846a-0e7c745a8cbe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09c2c3d3-9afa-4303-846a-0e7c745a8cbe\",\n \"name\": \"09c2c3d3-9afa-4303-846a-0e7c745a8cbe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-19T17:47:14.335034804Z\",\n \"endTime\": \"2026-03-19T17:49:12.841643046Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Failed to get virtual network 'customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-bq8pz8/providers/Microsoft.Network/virtualNetworks/customer-vnet\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-bq8pz8\\\" to finish creating: GET https://rp.j1746944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09c2c3d3-9afa-4303-846a-0e7c745a8cbe\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09c2c3d3-9afa-4303-846a-0e7c745a8cbe\\\",\\n \\\"name\\\": \\\"09c2c3d3-9afa-4303-846a-0e7c745a8cbe\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-19T17:47:14.335034804Z\\\",\\n \\\"endTime\\\": \\\"2026-03-19T17:49:12.841643046Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-bq8pz8/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000f66380\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-bq8pz8\\\" to finish creating: GET https://rp.j1746944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09c2c3d3-9afa-4303-846a-0e7c745a8cbe\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09c2c3d3-9afa-4303-846a-0e7c745a8cbe\\\",\\n \\\"name\\\": \\\"09c2c3d3-9afa-4303-846a-0e7c745a8cbe\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-19T17:47:14.335034804Z\\\",\\n \\\"endTime\\\": \\\"2026-03-19T17:49:12.841643046Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-bq8pz8/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.m...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4542/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034666434281345024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4542,"pr_state":"closed","pr_sha":"b0991e867d40790061a8cc08bdc3741fb90a9bef","final_merged_sha":"52fc7d3a41cdc9d6774bfe7f5f45b5768abaf818","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T16:20:54Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"5ea5407b173315a2a1c63a0be03e5e3cae7fa84e855e008380c096c7f406abed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4542/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034666434281345024","occurred_at":"2026-03-19T16:20:54Z","failed_at":"e2e","signature_id":"2b0ca2ad054bf7715c5c142f5ef455fb85023382c03c1b1c6b7ca7aee5d18876","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"parkunlock(...)\n\t/usr/local/go/src/runtime/proc.go:466\nruntime.semacquire1(0xc00104b6d8, 0x0, 0x1, 0x0, 0x19)\n\t/usr/local/go/src/runtime/sema.go:192 +0x21d fp=0xc000bab5f8 sp=0xc000bab590 pc=0x2aaa3dd\nsync.runtime_SemacquireWaitGroup(0xc00079f540?, 0x40?)\n\t/usr/local/go/src/runtime/sema.go:114 +0x2e fp=0xc000bab630 sp=0xc000bab5f8 pc=0x2acf9ae\nsync.(*WaitGroup).Wait(0xc00104b6d0)\n\t/usr/local/go/src/sync/waitgroup.go:206 +0x85 fp=0xc000bab658 sp=0xc000bab630 pc=0x2af2e05\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups(0xc0008b4a20, {0x7fcd46f0ce60, 0xc001037ec0}, {{0xc0008a1450, 0x1, 0x1}, 0x34630b8a000, {0x782a6c2, 0x8}})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:305 +0x273 fp=0xc000bab768 sp=0xc000bab658 pc=0x4ec41f3\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources(0xc0008b4a20, {0x7fcd46f0ce60, 0xc001037ec0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:224 +0x225 fp=0xc000bab888 sp=0xc000bab768 pc=0x4ec3a85\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources-fm({0x7fcd46f0ce60?, 0xc001037ec0?})\n\t\u003cautogenerated\u003e:1 +0x33 fp=0xc000bab8b0 sp=0xc000bab888 pc=0x4ed1c13\nruntime.call16(0xc0012c4c00, 0xc0002c0fe0, 0x0, 0x0, 0x0, 0x10, 0xc000babb08)\n\t/usr/local/go/src/runtime/asm_amd64.s:774 +0x43 fp=0xc000bab8d0 sp=0xc000bab8b0 pc=0x2ad4e83\nruntime.reflectcall(0x7289c80?, 0xc001037ec0?, 0x2?, 0x785af6d?, 0x0?, 0x12?, 0x6ed16a0?)\n\t\u003cautogenerated\u003e:1 +0x36 fp=0xc000bab910 sp=0xc000bab8d0 pc=0x2add716\nreflect.Value.call({0x6867740?, 0xc0002c0fe0?, 0xc0011bc5b0?}, {0x781f521, 0x4}, {0xc000d5df60, 0x1, 0x0?})\n\t/usr/local/go/src/reflect/value.go:581 +0xcc6 fp=0xc000babe70 sp=0xc000bab910 pc=0x2b106e6\nreflect.Value.Call({0x6867740?, 0xc0002c0fe0?, 0xc0011bc5b0?}, {0xc000d5df60?, 0xc000d5df28?, 0x2ace28e?})\n\t/usr/local/go/src/reflect/value.go:365 +0xb9 fp=0xc000babee8 sp=0xc000babe70 pc=0x2b0f7d9\ngithub.com/onsi/ginkgo/v2/internal.NewCleanupNode.func2({0x827b940?, 0xc001037ec0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:589 +0x150 fp=0xc000babf88 sp=0xc000babee8 pc=0x2f00ef0\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc000babfe0 sp=0xc000babf88 pc=0x2f139bb\nruntime.goexit({})\n\t/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000babfe8 sp=0xc000babfe0 pc=0x2ad6921\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 85\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\ngoroutine 1289 gp=0xc001601dc0 m=nil [select]:\nruntime.gopark(0xc000d67ab8?, 0x2?, 0xa0?, 0x22?, 0xc000d67aac?)\n\t/usr/local/go/src/runtime/proc.go:460 +0xce fp=0xc000d67938 sp=0xc000d67918 pc=0x2ace28e\nruntime.selectgo(0xc000d67ab8, 0xc000d67aa8, 0xc0005d2a80?, 0x0, 0x329dd3c?, 0x1)\n\t/usr/local/go/src/runtime/select.go:351 +0x8b7 fp=0xc000d67a78 sp=0xc000d67938 pc=0x2aa9137\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8266038, 0xc000b89ea0}, 0x783ee71?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36 +0x76 fp=0xc000d67ae8 sp=0xc000d67a78 pc=0x329d496\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x82aa0a0, {0x8266038, 0xc000b89ea0}, 0xc0016490a0)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284 +0x732 fp=0xc000d67de8 sp=0xc000d67ae8 pc=0x36d4232\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster({0x8266038, 0xc000b89d60}, 0xc0016ca180, {0xc0009d4b60, 0x1c}, {0xc0015f0920, 0x1b}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:198 +0x218 fp=0xc000d67ef0 sp=0xc000d67de8 pc=0x4eaf118\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:316 +0x94 fp=0xc000d67f78 sp=0xc000d67ef0 pc=0x4eb0f34\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc000d67fe0 sp=0xc000d67f78 pc=0x376b450\nruntime.goexit({})\n\t/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000d67fe8 sp=0xc000d67fe0 pc=0x2ad6921\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1332\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\nrax 0xca\nrbx 0x0\nrcx 0x2ad8743\nrdx 0x0\nrdi 0xb364af0\nrsi 0x80\nrbp 0x7fff66157880\nrsp 0x7fff66157838\nr8 0x0\nr9 0x0\nr10 0x0\nr11 0x286\nr12 0xc0000a0008\nr13 0x1\nr14 0xb362a00\nr15 0x5\nrip 0x2ad8741\nrflags 0x286\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command\nparkunlock(...)\n\t/usr/local/go/src/runtime/proc.go:466\nruntime.semacquire1(0xc00104b6d8, 0x0, 0x1, 0x0, 0x19)\n\t/usr/local/go/src/runtime/sema.go:192 +0x21d fp=0xc000bab5f8 sp=0xc000bab590 pc=0x2aaa3dd\nsync.runtime_SemacquireWaitGroup(0xc00079f540?, 0x40?)\n\t/usr/local/go/src/runtime/sema.go:114 +0x2e fp=0xc000bab630 sp=0xc000bab5f8 pc=0x2acf9ae\nsync.(*WaitGroup).Wait(0xc00104b6d0)\n\t/usr/local/go/src/sync/waitgroup.go:206 +0x85 fp=0xc000bab658 sp=0xc000bab630 pc=0x2af2e05\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups(0xc0008b4a20, {0x7fcd46f0ce60, 0xc001037ec0}, {{0xc0008a1450, 0x1, 0x1}, 0x34630b8a000, {0x782a6c2, 0x8}})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:305 +0x273 fp=0xc000bab768 sp=0xc000bab658 pc=0x4ec41f3\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources(0xc0008b4a20, {0x7fcd46f0ce60, 0xc001037ec0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:224 +0x225 fp=0xc000bab888 sp=0xc000bab768 pc=0x4ec3a85\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources-fm({0x7fcd46f0ce60?, 0xc001037ec0?})\n\t\u003cautogenerated\u003e:1 +0x33 fp=0xc000bab8b0 sp=0xc000bab888 pc=0x4ed1c13\nruntime.call16(0xc0012c4c00, 0xc0002c0fe0, 0x0, 0x0, 0x0, 0x10, 0xc000babb08)\n\t/usr/local/go/src/runtime/asm_amd64.s:774 +0x43 fp=0xc000bab8d0 sp=0xc000bab8b0 pc=0x2ad4e83\nruntime.reflectcall(0x7289c80?, 0xc001037ec0?, 0x2?, 0x785af6d?, 0x0?, 0x12?, 0x6ed16a0?)\n\t\u003cautogenerated\u003e:1 +0x36 fp=0xc000bab910 sp=0xc000bab8d0 pc=0x2add716\nreflect.Value.call({0x6867740?, 0xc0002c0fe0?, 0xc0011bc5b0?}, {0x781f521, 0x4}, {0xc000d5df60, 0x1, 0x0?})\n\t/usr/local/go/src/reflect/value.go:581 +0xcc6 fp=0xc000babe70 sp=0xc000bab910 pc=0x2b106e6\nreflect.Value.Call({0x6867740?, 0xc0002c0fe0?, 0xc0011bc5b0?}, {0xc000d5df60?, 0xc000d5df28?, 0x2ace28e?})\n\t/usr/local/go/src/reflect/value.go:365 +0xb9 fp=0xc000babee8 sp=0xc000babe70 pc=0x2b0f7d9\ngithub.com/onsi/ginkgo/v2/internal.NewCleanupNode.func2({0x827b940?, 0xc001037ec0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:589 +0x150 fp=0xc000babf88 sp=0xc000babee8 pc=0x2f00ef0\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc000babfe0 sp=0xc000babf88 pc=0x2f139bb\nruntime.goexit({})\n\t/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000babfe8 sp=0xc000babfe0 pc=0x2ad6921\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 85\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\ngoroutine 1289 gp=0xc001601dc0 m=nil [select]:\nruntime.gopark(0xc000d67ab8?, 0x2?, 0xa0?, 0x22?, 0xc000d67aac?)\n\t/usr/local/go/src/runtime/proc.go:460 +0xce fp=0xc000d67938 sp=0xc000d67918 pc=0x2ace28e\nruntime.selectgo(0xc000d67ab8, 0xc000d67aa8, 0xc0005d2a80?, 0x0, 0x329dd3c?, 0x1)\n\t/usr/local/go/src/runtime/select.go:351 +0x8b7 fp=0xc000d67a78 sp=0xc000d67938 pc=0x2aa9137\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8266038, 0xc000b89ea0}, 0x783ee71?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36 +0x76 fp=0xc000d67ae8 sp=0xc000d67a78 pc=0x329d496\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x82aa0a0, {0x8266038, 0xc000b89ea0}, 0xc0016490a0)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284 +0x732 fp=0xc000d67de8 sp=0xc000d67ae8 pc=0x36d4232\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster({0x8266038, 0xc000b89d60}, 0xc0016ca180, {0xc0009d4b60, 0x1c}, {0xc0015f0920, 0x1b}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:198 +0x218 fp=0xc000d67ef0 sp=0xc000d67de8 pc=0x4eaf118\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:316 +0x94 fp=0xc000d67f78 sp=0xc000d67ef0 pc=0x4eb0f34\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc000d67fe0 sp=0xc000d67f78 pc=0x376b450\nruntime.goexit({})\n\t/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000d67fe8 sp=0xc000d67fe0 pc=0x2ad6921\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1332\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\nrax 0xca\nrbx 0x0\nrcx 0x2ad8743\nrdx 0x0\nrdi 0xb364af0\nrsi 0x80\nrbp 0x7fff66157880\nrsp 0x7fff66157838\nr8 0x0\nr9 0x0\nr10 0x0\nr11 0x286\nr12 0xc0000a0008\nr13 0x1\nr14 0xb362a00\nr15 0x5\nrip 0x2ad8741\nrflags 0x286\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4494/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034665648159723520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4494,"pr_state":"closed","pr_sha":"4586325c56abc947283239ac01caa8e3d425c21f","final_merged_sha":"b6d9c642ca814d40f9bb7032fbbf894bc4edfd69","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-19T16:17:47Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4536/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034661458565926912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4536,"pr_state":"closed","pr_sha":"833532cff602417be25a9c830f42ff2d65952aab","final_merged_sha":"833532cff602417be25a9c830f42ff2d65952aab","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-19T16:00:54Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"690b7bd1e4a0332651417043c03e021c949a767b7fdef12fda3ac9e6a334e011","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4536/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034661458565926912","occurred_at":"2026-03-19T16:00:54Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4494/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034652994686947328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4494,"pr_state":"closed","pr_sha":"330f2d01c5079b5a5f94d0e1d88f11eb152deb17","final_merged_sha":"b6d9c642ca814d40f9bb7032fbbf894bc4edfd69","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T15:27:30Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"206b497048be5ae9c561dbe6a4707b438e58ec92c9fd43207edb875bd07080ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4494/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034652994686947328","occurred_at":"2026-03-19T15:27:30Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4385/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034651324179550208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4385,"pr_state":"closed","pr_sha":"07e421bae19659fd95e6d16899bed8ad2fda3863","final_merged_sha":"07e421bae19659fd95e6d16899bed8ad2fda3863","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-19T15:20:52Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"b82f7b3957f4799490a3a8dac98058221f0607d46c98ca734f64d7c64e1098fc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4385/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034651324179550208","occurred_at":"2026-03-19T15:20:52Z","failed_at":"provision","signature_id":"eb42f536566c06969e79bb4dd530b90fade97f988fe3d18503444c2160414c7b","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/svc-mgmt-permissions","test_suite":"step graph","failure_text":"time=2026-03-19T16:02:24.737Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions description=\"Step svc-mgmt-permissions\\n Kind: ARM\\n Template: templates/svc-mgmt-permissions.bicep\\n Parameters: configurations/svc-mgmt-permissions.tmpl.bicepparam\"\ntime=2026-03-19T16:02:25.307Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions\ntime=2026-03-19T16:02:26.298Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions deployment=e52fd2d5066e5f609d2338831504b035a21018a9b956bf0f907eb07f86757b0f\ntime=2026-03-19T16:02:56.497Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9550208-mgmt-1/providers/Microsoft.Resources/deployments/e52fd2d5066e5f609d2338831504b035a21018a9b956bf0f907eb07f86757b0f/operationStatuses/08584276707397171591\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9550208-mgmt-1/providers/Microsoft.Resources/deployments/cx-backend-kv-access\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9550208-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-ert2g2qezajzq\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9550208-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-ert2g2qezajzq\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9550208-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-5rr4rbpkivz6y\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9550208-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-5rr4rbpkivz6y\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9550208-mgmt-1/providers/Microsoft.Resources/deployments/cx-access-5rr4rbpkivz6y\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9550208-mgmt-1/providers/Microsoft.Resources/deployments/cx-access-5rr4rbpkivz6y\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9550208-mgmt-1/providers/Microsoft.Resources/deployments/cx-access-ert2g2qezajzq\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9550208-mgmt-1/providers/Microsoft.Resources/deployments/cx-access-ert2g2qezajzq\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9550208-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-iqtgb5ozb5jju\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9550208-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-iqtgb5ozb5jju\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9550208-mgmt-1/providers/Microsoft.Resources/deployments/cx-access-iqtgb5ozb5jju\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9550208-mgmt-1/providers/Microsoft.Resources/deployments/cx-access-iqtgb5ozb5jju\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9550208-mgmt-1/providers/Microsoft.Resources/deployments/msi-refresher-msi-kv-access\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9550208-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-heazf5wzdhzaw\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9550208-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-heazf5wzdhzaw\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9550208-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-hyebjgkz4lbhe\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9550208-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-hyebjgkz4lbhe\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9550208-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-4m7g5555cws4s\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9550208-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-4m7g5555cws4s\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4508/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034650721747472384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4508,"pr_state":"closed","pr_sha":"a2ca195603032cc033a8e2ebe803918b756d5580","final_merged_sha":"4da0d9f5994ef9590af727a3ca8f65f2ef8646b8","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-19T15:18:28Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4541/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034650416997732352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4541,"pr_state":"closed","pr_sha":"364fb5c63fca1e0cbf62bd6496b150c1c7d401dd","final_merged_sha":"364fb5c63fca1e0cbf62bd6496b150c1c7d401dd","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-19T15:17:15Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"eda34e004a209425f90923b2a5c68c74c6c62a5368ac7be9de35dc1cfc50757d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4541/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034650416997732352","occurred_at":"2026-03-19T15:17:15Z","failed_at":"e2e","signature_id":"55e27180a48119aed7c921f055068dcca73d635eddc28835f6a67954d629aa92","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0011e6468\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-bbz7dj\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000884cc0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-bbz7dj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000884ca0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-bbz7dj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000884c80\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-bbz7dj\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0011e6468\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-bbz7dj\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000884cc0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-bbz7dj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000884ca0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-bbz7dj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000884c80\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-bbz7dj\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-xk2ks9\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034647207864438784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4393,"pr_state":"open","pr_sha":"50b09f4dff456732352028761f200f28d36d1c31","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-19T15:04:30Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4532/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034644636483457024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4532,"pr_state":"closed","pr_sha":"8033aacf4605ee2e3dd591bd5116c0e8c0a2b5f2","final_merged_sha":"8033aacf4605ee2e3dd591bd5116c0e8c0a2b5f2","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-19T14:54:14Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c9fd2a804202aa4b48647f142e2d4f2e19c7175ee9c8411b2125523529e28024","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4532/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034644636483457024","occurred_at":"2026-03-19T14:54:14Z","failed_at":"e2e","signature_id":"5949a963822127f3dad9b937684239701846caa02cf51be51482fa59f95676a7","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:89]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0007acfc0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-dwltsc\" to finish creating: GET https://rp.j3457024.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3935997-cedc-4c78-bded-528c72f0a956\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3935997-cedc-4c78-bded-528c72f0a956\",\n \"name\": \"e3935997-cedc-4c78-bded-528c72f0a956\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-19T15:38:49.709273653Z\",\n \"endTime\": \"2026-03-19T15:40:31.756262598Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-dwltsc/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-dwltsc\\\" to finish creating: GET https://rp.j3457024.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3935997-cedc-4c78-bded-528c72f0a956\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3935997-cedc-4c78-bded-528c72f0a956\\\",\\n \\\"name\\\": \\\"e3935997-cedc-4c78-bded-528c72f0a956\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-19T15:38:49.709273653Z\\\",\\n \\\"endTime\\\": \\\"2026-03-19T15:40:31.756262598Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-dwltsc/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0007acfa0\u003e{\n msg: \"failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-dwltsc\\\" to finish creating: GET https://rp.j3457024.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3935997-cedc-4c78-bded-528c72f0a956\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3935997-cedc-4c78-bded-528c72f0a956\\\",\\n \\\"name\\\": \\\"e3935997-cedc-4c78-bded-528c72f0a956\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-19T15:38:49.709273653Z\\\",\\n \\\"endTime\\\": \\\"2026-03-19T15:40:31.756262598Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-dwltsc/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds a...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:89]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0007acfc0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-dwltsc\" to finish creating: GET https://rp.j3457024.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3935997-cedc-4c78-bded-528c72f0a956\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3935997-cedc-4c78-bded-528c72f0a956\",\n \"name\": \"e3935997-cedc-4c78-bded-528c72f0a956\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-19T15:38:49.709273653Z\",\n \"endTime\": \"2026-03-19T15:40:31.756262598Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-dwltsc/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-dwltsc\\\" to finish creating: GET https://rp.j3457024.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3935997-cedc-4c78-bded-528c72f0a956\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3935997-cedc-4c78-bded-528c72f0a956\\\",\\n \\\"name\\\": \\\"e3935997-cedc-4c78-bded-528c72f0a956\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-19T15:38:49.709273653Z\\\",\\n \\\"endTime\\\": \\\"2026-03-19T15:40:31.756262598Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-dwltsc/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0007acfa0\u003e{\n msg: \"failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-dwltsc\\\" to finish creating: GET https://rp.j3457024.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3935997-cedc-4c78-bded-528c72f0a956\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3935997-cedc-4c78-bded-528c72f0a956\\\",\\n \\\"name\\\": \\\"e3935997-cedc-4c78-bded-528c72f0a956\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-19T15:38:49.709273653Z\\\",\\n \\\"endTime\\\": \\\"2026-03-19T15:40:31.756262598Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-dwltsc/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds a...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034641969069690880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4045,"pr_state":"open","pr_sha":"3207545fca598a0bbc926207da9f3ebd12d86873","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-19T14:43:39Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4540/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034641107874222080","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4540,"pr_state":"closed","pr_sha":"1a33c28cf367ed277888f36eeab14ec0d45afb1c","final_merged_sha":"1a33c28cf367ed277888f36eeab14ec0d45afb1c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-19T14:40:16Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4536/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034640805091610624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4536,"pr_state":"closed","pr_sha":"ac80b6f52327db1a6382f512027df625be36174c","final_merged_sha":"833532cff602417be25a9c830f42ff2d65952aab","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T14:39:04Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"957c76df41dff6b78ff74cb0ff67f92c890cc87fe990bac04f17fe6e15608a3e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4536/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034640805091610624","occurred_at":"2026-03-19T14:39:04Z","failed_at":"provision","signature_id":"8d6f68fae31acc214614ef21130c61c6cde2409c0bf162f7fb985221dce8244d","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-19T15:16:39.998Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-19T15:16:40.694Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:4837a862-3da1-46eb-b760-0862e107baf6 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j1610624] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j1610624.hcpsvc.osadev.cloud] image:map[digest:sha256:fb9048ef756c29182c06d3eb373b01e676ff3afbc048e973a65ff2c158d74c73 registry:arohcpsvcdev.azurecr.io repository:ci-op-w1qwzszk/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-w1qwzszk/pipeline:pull workloadIdentityClientId:20990b84-9470-46ed-bace-0a875eb04b3b workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:69bb32e9-27ee-4899-80e4-bff20dc9567d workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-03-19T15:16:40.694Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-19T15:16:40.938Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy namespace=sessiongate\ntime=2026-03-19T15:16:41.252Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy dryRun=false\ntime=2026-03-19T15:16:41.252Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-19T15:16:41.361Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-19T15:16:50.151Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF\"\ntime=2026-03-19T15:16:50.151Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-19T15:16:50.151Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-19T15:16:51.973Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy release=sessiongate namespace=sessiongate status=failed description=\"Release \\\"sessiongate\\\" failed: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF\" values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:4837a862-3da1-46eb-b760-0862e107baf6 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j1610624] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j1610624.hcpsvc.osadev.cloud] image:map[digest:sha256:fb9048ef756c29182c06d3eb373b01e676ff3afbc048e973a65ff2c158d74c73 registry:arohcpsvcdev.azurecr.io repository:ci-op-w1qwzszk/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-w1qwzszk/pipeline:pull workloadIdentityClientId:20990b84-9470-46ed-bace-0a875eb04b3b workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:69bb32e9-27ee-4899-80e4-bff20dc9567d workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-03-19T15:16:51.978Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5yUzW7aQBDH736KkU8gsRakTaRS5UCSSpUqVYhwStXDej2YDesda2YNIuLhKwcIDgUH5eaZnf3tfPw9kcMAjEIVGxS4hUwHHXTqsEPpM5rwy/psKIGtz3uwdf3WBb65cIn+1SOlNnt3908EABA/Ii+twZExVPkQ9yAWFLHkcx3wP3N7Z0Lu+EgZ8oHJOeRPXVIOdYas0KEJlvwZyAOWjtYFXprp2XhlkIOdWVN/M84YZb5NXS9EWQmWlPU5o8ieNTI8rpy7sz6zPq9Dy8o5lR7sEwmMqjAnti+6LmpMzpr16yPO0UoVGNgaUfJxIa0cT2F+PoVHNIxhzLS0GfK903VJ75shpmwt/b6SQMVkp8EHnFlvj6YkSQOYaCY1N2WiXyrGxFBRR77RXCUB+ZQgdt+q0F7nuJ9bU0J3J9vdVFL7QC5jHKnxA+buH7pMkq2TPOr+ScDP6XQ8oerEA+/N6O/3aFGlyB4Dyo96CUi0gdUcGSHYAiXoooQUwwrRQyfTAWt3J77qX92o/hc1+DYdXA8HN8Ovg6e4C0kCLTHX/ae424028EzWw8L67NZ6j9xYXeThsLGaq+p4R0UbKJnq00OiPSgp2wZfBumBo7wHBYrovEYSZ8iQrhvFZyjmXwAAAP//Qqqu7WAFAAA=\"\ntime=2026-03-19T15:16:51.981Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-19T15:16:51.982Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d8ac2e6cca70c7690a0f75cd38b523f3cb17701626a2f823018ba74b9a5ecb16","failure_pattern":"Internal error occurred: failed calling webhook \"validation.istio.io\": failed to call webhook: Post \"\u003curl\u003e\": EOF","search_query":"error running Helm release deployment Step, failed to deploy helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d8ac2e6cca70c7690a0f75cd38b523f3cb17701626a2f823018ba74b9a5ecb16"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034639154532323328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4444,"pr_state":"closed","pr_sha":"092800808fd635f2494b3d3e1993603df989962b","final_merged_sha":"092800808fd635f2494b3d3e1993603df989962b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-19T14:32:29Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4484/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034638810968494080","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4484,"pr_state":"closed","pr_sha":"c8ec508572a2236d05d09f8cb2d4e7925bed9775","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-19T14:31:08Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034634660788572160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4477,"pr_state":"closed","pr_sha":"29f5cfda8c5bd3a403e59176d85bc5a100ce5b26","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T14:13:00Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"6dee9b8ec20c3db26727376cbef5dce55fe9ea5f5179d878ed830a8a2e8bd4ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034634660788572160","occurred_at":"2026-03-19T14:13:00Z","failed_at":"e2e","signature_id":"14eb684f2f5a0538d6dfd4f52ed0e926c050b06b11326e8191bdeab8e98fa84e","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a96cc0\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-hv6bzb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-hv6bzb, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-hv6bzb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-hv6bzb, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ea27c0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-hv6bzb\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a96cc0\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-hv6bzb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-hv6bzb, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-hv6bzb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-hv6bzb, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ea27c0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-hv6bzb\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-hv6bzb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-hv6bzb, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4425/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034630225924460544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4425,"pr_state":"closed","pr_sha":"d4baf959cdc6fa9dfc2231df85bdf4fd186147cb","final_merged_sha":"855eea85f3052237fc8528c6d280c0ac41ee4307","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-19T13:57:02Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034626171307560960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4305,"pr_state":"closed","pr_sha":"5b57b30e904172a0f4f838b9b963c522b0932ba7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T13:40:55Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"5a00672106edf32568e1873bb8af0c8f2204f244a88c8adff9a3310fd2fe0b22","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034626171307560960","occurred_at":"2026-03-19T13:40:55Z","failed_at":"e2e","signature_id":"eeecca7bde168424d8819a49c8e04514b34e208d013a23c04ce2d33764bf9c32","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:73]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e05cc0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-update-nodes-hcp-cluster-l922hd\" at subscription scope to finish: stream error: stream ID 25; CANCEL; received from peer\n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-np-update-nodes-hcp-cluster-l922hd\\\" at subscription scope to finish: stream error: stream ID 25; CANCEL; received from peer\",\n err: \u003c*fmt.wrapError | 0xc000e05c80\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-np-update-nodes-hcp-cluster-l922hd\\\" at subscription scope to finish: stream error: stream ID 25; CANCEL; received from peer\",\n err: \u003c*fmt.wrapError | 0xc000e05c60\u003e{\n msg: \"failed waiting for deployment \\\"mi-np-update-nodes-hcp-cluster-l922hd\\\" at subscription scope to finish: stream error: stream ID 25; CANCEL; received from peer\",\n err: \u003chttp2.StreamError\u003e{\n StreamID: 25,\n Code: 8,\n Cause: \u003c*errors.errorString | 0xb1bc5c0\u003e{\n s: \"received from peer\",\n },\n },\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:73]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e05cc0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-update-nodes-hcp-cluster-l922hd\" at subscription scope to finish: stream error: stream ID 25; CANCEL; received from peer\n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-np-update-nodes-hcp-cluster-l922hd\\\" at subscription scope to finish: stream error: stream ID 25; CANCEL; received from peer\",\n err: \u003c*fmt.wrapError | 0xc000e05c80\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-np-update-nodes-hcp-cluster-l922hd\\\" at subscription scope to finish: stream error: stream ID 25; CANCEL; received from peer\",\n err: \u003c*fmt.wrapError | 0xc000e05c60\u003e{\n msg: \"failed waiting for deployment \\\"mi-np-update-nodes-hcp-cluster-l922hd\\\" at subscription scope to finish: stream error: stream ID 25; CANCEL; received from peer\",\n err: \u003chttp2.StreamError\u003e{\n StreamID: 25,\n Code: 8,\n Cause: \u003c*errors.errorString | 0xb1bc5c0\u003e{\n s: \"received from peer\",\n },\n },\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1ecf05768a7a3a07564c3ccbb279c2f05f6758e134686b4c9da3bae0b1c444b4","failure_pattern":"failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-update-nodes-hcp-cluster-\u003cid\u003e\" at subscription scope to finish: stream error: stream ID 25; CANCEL; received from peer","search_query":"failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-update-nodes-hcp-cluster-l922hd\" at subscription scope to finish: stream error: stream ID 25; CANCEL; received from peer"}},{"row_id":"720073114de563b46cc8b0a8508c1d7ef5683d33eb54224ace21b0ccdeb89904","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034626171307560960","occurred_at":"2026-03-19T13:40:55Z","failed_at":"e2e","signature_id":"1e1416d71c8377df5dadfa2cdb51072517a7c776bc774cf87f23cd531fc84ea6","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:116]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b0fba0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-5fpjrx\": PUT https://rp.j7560960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-5fpjrx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-5fpjrx\\\": PUT https://rp.j7560960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-5fpjrx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000b0fb80\u003e{\n msg: \"failed starting cluster creation \\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-5fpjrx\\\": PUT https://rp.j7560960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-5fpjrx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0014f4840\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Thu, 19 Mar 2026 14:27:15 GMT\",\n ],\n \"Content-Length\": [\"107\"],\n \"X-Envoy-Upstream-Service-Time\": [\"30\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"ed70a8b3-fa89-4990-a434-45df8d77753e\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000b0fa40\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j7560960.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-5fpjrx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\",\n RawPath: \"\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:116]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b0fba0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-5fpjrx\": PUT https://rp.j7560960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-5fpjrx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-5fpjrx\\\": PUT https://rp.j7560960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-5fpjrx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000b0fb80\u003e{\n msg: \"failed starting cluster creation \\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-5fpjrx\\\": PUT https://rp.j7560960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-5fpjrx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0014f4840\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Thu, 19 Mar 2026 14:27:15 GMT\",\n ],\n \"Content-Length\": [\"107\"],\n \"X-Envoy-Upstream-Service-Time\": [\"30\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"ed70a8b3-fa89-4990-a434-45df8d77753e\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000b0fa40\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j7560960.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-5fpjrx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\",\n RawPath: \"\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["1ecf05768a7a3a07564c3ccbb279c2f05f6758e134686b4c9da3bae0b1c444b4","223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4532/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034618406736498688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4532,"pr_state":"closed","pr_sha":"93bfa810033898ff55f0b3cfd5e2d1e839532e3f","final_merged_sha":"8033aacf4605ee2e3dd591bd5116c0e8c0a2b5f2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T13:10:04Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ccc394fa9c158f73f0e9756a473bda938415257774d8d36de62b400b8d12ecdc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4532/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034618406736498688","occurred_at":"2026-03-19T13:10:04Z","failed_at":"provision","signature_id":"ff880cfb138e240d4139e64886fbd41eea79777c725e5bc81c662e5a060e45de","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-19T13:47:01.252Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-19T13:47:01.909Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[armPermissionsManagerIdentity:map[certName:armHelperCert2 clientId:3331e670-0804-48e8-a086-6241671ddc93 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj6498688.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj6498688.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:6d9b1a79-b2f3-40b1-a69e-42423f2a74b9 databaseName:arohcpprow-rp-j6498688 databaseUrl:https://arohcpprow-rp-j6498688.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-h2tti4cs/pipeline@sha256:d4c0427893bf6e5995ed37f2b00c3119b0fe24b83fa546f811ad51c4c3d8a2f1 replicas:2 zoneCount:4] exitOnPanic:true fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] miMock:map[certName:msiMockCert2 clientId:e8723db7-9b9e-46a4-9f7d-64d75c3534f0 principalId:d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-h2tti4cs/pipeline:pull workloadIdentityClientId:052b85f6-11d4-4120-8d5c-2826936d6ac9 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:6d9b1a79-b2f3-40b1-a69e-42423f2a74b9 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-19T13:47:01.909Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-19T13:47:01.965Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-03-19T13:47:02.071Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-03-19T13:47:02.071Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-19T13:47:02.177Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-19T13:52:06.435Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-19T13:52:06.435Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-19T13:52:06.435Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-19T13:52:10.776Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description=\"Release \\\"aro-hcp-backend-dev\\\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[armPermissionsManagerIdentity:map[certName:armHelperCert2 clientId:3331e670-0804-48e8-a086-6241671ddc93 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj6498688.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj6498688.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:6d9b1a79-b2f3-40b1-a69e-42423f2a74b9 databaseName:arohcpprow-rp-j6498688 databaseUrl:https://arohcpprow-rp-j6498688.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-h2tti4cs/pipeline@sha256:d4c0427893bf6e5995ed37f2b00c3119b0fe24b83fa546f811ad51c4c3d8a2f1 replicas:2 zoneCount:4] exitOnPanic:true fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] miMock:map[certName:msiMockCert2 clientId:e8723db7-9b9e-46a4-9f7d-64d75c3534f0 principalId:d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-h2tti4cs/pipeline:pull workloadIdentityClientId:052b85f6-11d4-4120-8d5c-2826936d6ac9 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:6d9b1a79-b2f3-40b1-a69e-42423f2a74b9 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-19T13:52:10.779Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/4yTwW7bPBCE73qKhU42IAaO8/8p6iKHNO2pSGE0PaXogSKnMmOKKywpGw788IXsuBYMufV1dr7hLrGbeSQSRG7FINIdWZ100qXHiMsXmPTFBTuLSVyoCtpLX3WNPxJWCDslNtoc5PGPjIgof+Dwy1WPuskLykttlghW6ddWoKQNydVQZmfp6lpYLUyTF39hz7q/sUcne2gLUfAwyXE44/zognWh6if/G7w3Mm+9H2Cb1ntVHvVTsE0LFvequ+A5e2c2O5f3vFY1kjgT1VvWAP4EWTmDXuVgPsAD0Cc0njc1QhrgBvxzQLo+EZIz+uQHLnjvCUaQ5sIrZyEPXsfY/6G4n0EtsVEr3fp0fs5HDi6xXNb2G3NvDLf7UQe82c8P2bItIQEJ8XO3sDHb0noBAXVLGJOuGyqR1kCgkdUJnTzKp5PprZrcqOv3369vZv+9m02mz/mYrq7onOf/6Wxy+5yPx9mWXtgFWrpg71wIkN6ZcaDjdfXP6vSesi01wl312GhBDdu9+bKQgjxXBdWIUVddJIuFULnpDW8Rze8AAAD///VVd6oMBAAA\"\ntime=2026-03-19T13:52:10.783Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-19T13:52:10.784Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f70296c5b0bdd447fa07f45b50524b3847ac3efdd92659ec6c4198951f8a52ee","failure_pattern":"resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f70296c5b0bdd447fa07f45b50524b3847ac3efdd92659ec6c4198951f8a52ee"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4539/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034617751598796800","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4539,"pr_state":"closed","pr_sha":"87a94b60840c1bd78c84378b323f1a824af5a7c0","final_merged_sha":"e4ce686fe32b144e5b46173e80928cf3e0dc3359","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-19T13:07:27Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4514/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034614441496023040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4514,"pr_state":"closed","pr_sha":"61e704d766a95da5a14cf10c199fb79215449bb4","final_merged_sha":"61e704d766a95da5a14cf10c199fb79215449bb4","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-19T12:54:18Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034611758227787776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4347,"pr_state":"closed","pr_sha":"a82c62541bf569a06efb0a6211c6ae9e8a2c01a4","final_merged_sha":"a82c62541bf569a06efb0a6211c6ae9e8a2c01a4","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-19T12:43:39Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"de2a6833702dd3e5007b8b1f15ac72bf8a093f43011286a70e52e87be011c3ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034611758227787776","occurred_at":"2026-03-19T12:43:39Z","failed_at":"e2e","signature_id":"6a9e8e2ae9676063524ba31ea01cfa88619be6e03a7fd8edf5f6cbc70ab0234d","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:133]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00104e5a0\u003e: \n failed to create NodePool oidc-wi-np, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool oidc-wi-np in resource group oidc-wi-49ztl8, error: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-49ztl8\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool oidc-wi-np, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool oidc-wi-np in resource group oidc-wi-49ztl8, error: failed waiting for nodepool=\\\"oidc-wi-np\\\" for cluster \\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-49ztl8\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001405ee0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool oidc-wi-np in resource group oidc-wi-49ztl8\",\n },\n \u003c*fmt.wrapError | 0xc00093e1e0\u003e{\n msg: \"failed waiting for nodepool=\\\"oidc-wi-np\\\" for cluster \\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-49ztl8\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:133]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00104e5a0\u003e: \n failed to create NodePool oidc-wi-np, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool oidc-wi-np in resource group oidc-wi-49ztl8, error: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-49ztl8\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool oidc-wi-np, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool oidc-wi-np in resource group oidc-wi-49ztl8, error: failed waiting for nodepool=\\\"oidc-wi-np\\\" for cluster \\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-49ztl8\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001405ee0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool oidc-wi-np in resource group oidc-wi-49ztl8\",\n },\n \u003c*fmt.wrapError | 0xc00093e1e0\u003e{\n msg: \"failed waiting for nodepool=\\\"oidc-wi-np\\\" for cluster \\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-49ztl8\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool oidc-wi-np in resource group oidc-wi-49ztl8, error: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-49ztl8\" to finish creating: context deadline exceeded"}},{"row_id":"f804e3692d8ae2596a1e1fbe900e128e3e95631356c1ea06aa144afc1649d8cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034611758227787776","occurred_at":"2026-03-19T12:43:39Z","failed_at":"e2e","signature_id":"d136a65f86fae2b63ce2ee5ee97be16b4f0101988b37aae2b61a5c8e70d581ca","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:281]: Timed out after 30.001s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000b21780\u003e: \n read access verification failed: get pods -n kube-system: pods is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"pods\" in API group \"\" in the namespace \"kube-system\"\n get configmaps -n kube-system: configmaps is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"configmaps\" in API group \"\" in the namespace \"kube-system\"\n {\n msg: \"read access verification failed: get pods -n kube-system: pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\\nget configmaps -n kube-system: configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.joinError | 0xc000e6c2a0\u003e{\n errs: [\n \u003c*fmt.wrapError | 0xc000b21480\u003e{\n msg: \"get pods -n kube-system: pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.StatusError | 0xc00057a140\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n Reason: \"Forbidden\",\n Details: {Name: \"\", Group: \"\", Kind: \"pods\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n },\n },\n \u003c*fmt.wrapError | 0xc000b21720\u003e{\n msg: \"get configmaps -n kube-system: configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.StatusError | 0xc00057bb80\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n Reason: \"Forbidden\",\n Details: {Name: \"\", Group: \"\", Kind: \"configmaps\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n },\n },\n ],\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:281]: Timed out after 30.001s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000b21780\u003e: \n read access verification failed: get pods -n kube-system: pods is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"pods\" in API group \"\" in the namespace \"kube-system\"\n get configmaps -n kube-system: configmaps is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"configmaps\" in API group \"\" in the namespace \"kube-system\"\n {\n msg: \"read access verification failed: get pods -n kube-system: pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\\nget configmaps -n kube-system: configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.joinError | 0xc000e6c2a0\u003e{\n errs: [\n \u003c*fmt.wrapError | 0xc000b21480\u003e{\n msg: \"get pods -n kube-system: pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.StatusError | 0xc00057a140\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n Reason: \"Forbidden\",\n Details: {Name: \"\", Group: \"\", Kind: \"pods\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n },\n },\n \u003c*fmt.wrapError | 0xc000b21720\u003e{\n msg: \"get configmaps -n kube-system: configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.StatusError | 0xc00057bb80\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n Reason: \"Forbidden\",\n Details: {Name: \"\", Group: \"\", Kind: \"configmaps\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n },\n },\n ],\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3c6a69913a4e7e3ddde30c4ad0c3578c4319954e53163360fc7432ced31dda4a","failure_pattern":"read access verification failed: get pod \u003cpod\u003e kube-system: pods is forbidden: User \"system:sre-break-glass:\u003cuuid\u003e\" cannot list resource \"pods\" in API group \"\" in the namespace \"\u003cnamespace\u003e\"","search_query":"read access verification failed: get pods -n kube-system: pods is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"pods\" in API group \"\" in the namespace \"kube-system\""}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["3c6a69913a4e7e3ddde30c4ad0c3578c4319954e53163360fc7432ced31dda4a","a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4517/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034607266602160128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4517,"pr_state":"closed","pr_sha":"98f9b5f64bc2a1090552d4cecf1ed332bfca4775","final_merged_sha":"98f9b5f64bc2a1090552d4cecf1ed332bfca4775","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-19T12:25:48Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034606884459122688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4529,"pr_state":"closed","pr_sha":"b1f9c4a6d0f13c973a84bb6ed80d5b39ab7e0773","final_merged_sha":"b1f9c4a6d0f13c973a84bb6ed80d5b39ab7e0773","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-19T12:24:16Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034603431594823680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4444,"pr_state":"closed","pr_sha":"092800808fd635f2494b3d3e1993603df989962b","final_merged_sha":"092800808fd635f2494b3d3e1993603df989962b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-19T12:10:33Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4537/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034601390210289664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4537,"pr_state":"closed","pr_sha":"71e53db487f9156c76b83e23dbed8f6c749c4bb1","final_merged_sha":"71e53db487f9156c76b83e23dbed8f6c749c4bb1","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-19T12:02:09Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4531/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034600502456487936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4531,"pr_state":"closed","pr_sha":"e3f0945f33d4cde7c0df12f230b387f72c667e3f","final_merged_sha":"e3f0945f33d4cde7c0df12f230b387f72c667e3f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-19T11:58:55Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4533/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034600228711043072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4533,"pr_state":"closed","pr_sha":"212f334cfaf06e962ef8ce0768aed92badda92c9","final_merged_sha":"212f334cfaf06e962ef8ce0768aed92badda92c9","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-19T11:57:50Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034596001032966144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4503,"pr_state":"closed","pr_sha":"16b44df65e798f2ce166f58b4b3cdc45fae3ccec","final_merged_sha":"3a9b5c9cb4bcc81a28d2a98b1dd50a3c686f225a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T11:41:02Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"10d43ea57a65c2a3ed2ef4045577b2d06c1225c6d6e4d8cbeffd477859408b82","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034596001032966144","occurred_at":"2026-03-19T11:41:02Z","failed_at":"e2e","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:35]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:35]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4481/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034592415666409472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4481,"pr_state":"closed","pr_sha":"a0ece52bde4e4930997796ae17d728b08f758b6b","final_merged_sha":"a0ece52bde4e4930997796ae17d728b08f758b6b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-19T11:26:47Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4535/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034590914676002816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4535,"pr_state":"closed","pr_sha":"3f902fae0079c098423ae1140a498f982a287e3f","final_merged_sha":"3f902fae0079c098423ae1140a498f982a287e3f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-19T11:20:49Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4517/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034590843658047488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4517,"pr_state":"closed","pr_sha":"6d35b73d52257ae7a5bbc02428187d621caa6d9a","final_merged_sha":"98f9b5f64bc2a1090552d4cecf1ed332bfca4775","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T11:20:32Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d92257db4652084afb6bf57b6529be03447945bf566167ed5e807cd477452fa7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4517/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034590843658047488","occurred_at":"2026-03-19T11:20:32Z","failed_at":"provision","signature_id":"9df96cedd304ce960ec46d168dc0a0b5c62065ee44735bac356e14225c792ada","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Agent/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-03-19T11:58:29.075Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=global step=mirror-image description=\"Step mirror-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e maestro.image.registry \u003cnil\u003e}:{\u003cnil\u003e maestro.image.repository \u003cnil\u003e}@{\u003cnil\u003e maestro.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-19T11:59:12.604Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org\\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nFailed to log in to ACR arohcpsvcdev: ERROR: The command failed with an unexpected error. Here is the traceback:\\nERROR: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /oauth2/exchange (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 204, in _new_conn\\n sock = connection.create_connection(\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/util/connection.py\\\", line 60, in create_connection\\n for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/python3.12/socket.py\\\", line 963, in getaddrinfo\\n for res in _socket.getaddrinfo(host, port, family, type, proto, flags):\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\nsocket.gaierror: [Errno -2] Name or service not known\\n\\nThe above exception was the direct cause of the following exception:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 787, in urlopen\\n response = self._make_request(\\n ^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 488, in _make_request\\n raise new_e\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 464, in _make_request\\n self._validate_conn(conn)\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 1093, in _validate_conn\\n conn.connect()\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 759, in connect\\n self.sock = sock = self._new_conn()\\n ^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 211, in _new_conn\\n raise NameResolutionError(self.host, self, e) from e\\nurllib3.exceptions.NameResolutionError: HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\n\\nThe above exception was the direct cause of the following exception:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/adapters.py\\\", line 667, in send\\n resp = conn.urlopen(\\n ^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 841, in urlopen\\n retries = retries.increment(\\n ^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/util/retry.py\\\", line 535, in increment\\n raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\nurllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /oauth2/exchange (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\n\\nDuring handling of the above exception, another exception occurred:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/knack/cli.py\\\", line 233, in invoke\\n cmd_result = self.invocation.execute(args)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 682, in execute\\n raise ex\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 812, in _run_jobs_serially\\n results.append(self._run_job(expanded_arg, cmd_copy))\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 781, in _run_job\\n result = cmd_copy(params)\\n ^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 336, in __call__\\n return self.handler(*args, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/command_operation.py\\\", line 120, in handler\\n return op(**command_args)\\n ^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/custom.py\\\", line 294, in acr_login\\n login_server, _, password = get_login_credentials(\\n ^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 505, in get_login_credentials\\n return _get_credentials(cmd,\\n ^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 449, in _get_credentials\\n return login_server, EMPTY_GUID, _get_aad_token(cli_ctx,\\n ^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 285, in _get_aad_token\\n return _get_aad_token_after_challenge(cli_ctx,\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 159, in _get_aad_token_after_challenge\\n response = requests.post(url=authhost, data=urlencode(content), headers=headers,\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/api.py\\\", line 115, in post\\n return request(\\\"post\\\", url, data=data, json=json, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/api.py\\\", line 59, in request\\n return session.request(method=method, url=url, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/sessions.py\\\", line 589, in request\\n resp = self.send(prep, **send_kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/sessions.py\\\", line 703, in send\\n r = adapter.send(request, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/adapters.py\\\", line 700, in send\\n raise ConnectionError(e, request=request)\\nrequests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /oauth2/exchange (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\nTo check existing issues, please visit: https://github.com/Azure/azure-cli/issues\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9a2a4cf3344f5eab04c519ebc438e0419dbf08c2ef1692b2ed6fd7228e53ded3","failure_pattern":"requests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /oauth2/exchange (Caused by NameResolutionError(\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve","search_query":"requests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /oauth2/exchange (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9a2a4cf3344f5eab04c519ebc438e0419dbf08c2ef1692b2ed6fd7228e53ded3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034590487196733440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4347,"pr_state":"closed","pr_sha":"a82c62541bf569a06efb0a6211c6ae9e8a2c01a4","final_merged_sha":"a82c62541bf569a06efb0a6211c6ae9e8a2c01a4","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-19T11:19:07Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"f481dcab0bb4250ff541105a8ed63032d617720902751214bd90fd68039e241d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034590487196733440","occurred_at":"2026-03-19T11:19:07Z","failed_at":"provision","signature_id":"978ff2531d516c7460803bc6f35f25be48cfe39fa3c6bb68f641e5def29baec8","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-19T11:38:52.390Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-19T11:38:55.084Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-3640380727.bicepparam level=Error range=L20:32-L20:33 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T11:38:55.084Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-3640380727.bicepparam level=Error range=L20:32-L20:33 code=BCP019\ntime=2026-03-19T11:38:55.084Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-3640380727.bicepparam level=Error range=L21:31-L21:32 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T11:38:55.084Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-3640380727.bicepparam level=Error range=L21:31-L21:32 code=BCP019\ntime=2026-03-19T11:38:55.084Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e","failure_pattern":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors","search_query":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034590470616649728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4403,"pr_state":"open","pr_sha":"19a81cb5753781fff039fc75814c0f5c9b29436a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T11:19:03Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ed825836dc0c635152f3443dc640a72396e8904121ff4d5626af341f27777c93","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034590470616649728","occurred_at":"2026-03-19T11:19:03Z","failed_at":"provision","signature_id":"c027c4d12f6a3d02006d0282f3d5b1020328ed77f7bf43d76c7a5e7566e656ac","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-03-19T11:43:03.289Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-03-19T11:43:06.102Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-03-19T11:43:07.629Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=987cd1e56b36c548df3015be5c928fe6488455c5a3a725cd3294e876ba42e4e1\ntime=2026-03-19T11:55:40.655Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j6649728-svc/providers/Microsoft.Resources/deployments/987cd1e56b36c548df3015be5c928fe6488455c5a3a725cd3294e876ba42e4e1/operationStatuses/08584276862987044363\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6649728-svc/providers/Microsoft.Resources/deployments/profile-begm52m5pz2do\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6649728-svc/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j6649728-svc/resourceAssociations/nsp-j6649728-svc-ra-437d90f1-5adb-519e-85e4-35742f0071c8\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AnmPreconditionFailedException\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Error while processing request for association /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6649728-svc/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j6649728-svc/resourceAssociations/nsp-j6649728-svc-ra-437d90f1-5adb-519e-85e4-35742f0071c8, Error message: There is already an operation in progress which requires exclusive lock on this service arohcpprow-rp-j6649728. Please retry the operation after sometime.\\\\\\\\r\\\\\\\\nActivityId: 37f4ab3a-4e0e-4a4a-8cd6-156f2d6abdc2, Microsoft.Azure.Documents.Common/2.14.0\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0f53a8239a3c5bbc9779fb8310071a301a0db32a497288816e31d1e2b5794349","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AnmPreconditionFailedException; detail message Error while processing network security perimeter association.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0f53a8239a3c5bbc9779fb8310071a301a0db32a497288816e31d1e2b5794349"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4532/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034587058529374208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4532,"pr_state":"closed","pr_sha":"93bfa810033898ff55f0b3cfd5e2d1e839532e3f","final_merged_sha":"8033aacf4605ee2e3dd591bd5116c0e8c0a2b5f2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T11:05:30Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"e8a8fa0e7837bbc5d78ccec85849c1ca16f273759bd81517172402cf4990d2ac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4532/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034587058529374208","occurred_at":"2026-03-19T11:05:30Z","failed_at":"provision","signature_id":"80acc19cff95c7d8aebc9ad617470ece2c1144189212496cd75c3ab5bd9cb934","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-19T11:21:21.535Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-19T11:21:23.815Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1463722299.bicepparam level=Error range=L20:32-L20:33 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T11:21:23.815Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1463722299.bicepparam level=Error range=L20:32-L20:33 code=BCP019\ntime=2026-03-19T11:21:23.815Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1463722299.bicepparam level=Error range=L21:31-L21:32 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T11:21:23.816Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1463722299.bicepparam level=Error range=L21:31-L21:32 code=BCP019\ntime=2026-03-19T11:21:23.816Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e","failure_pattern":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors","search_query":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4494/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034578000837611520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4494,"pr_state":"closed","pr_sha":"e33c0c332280f77da97fe94a010febf32c9b7f86","final_merged_sha":"b6d9c642ca814d40f9bb7032fbbf894bc4edfd69","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T10:29:30Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"2376461a83aab5f5e19d83b1f84033e59d7fffe60c17afb6e5806c6a22df9746","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4494/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034578000837611520","occurred_at":"2026-03-19T10:29:30Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034574367182032896","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4403,"pr_state":"open","pr_sha":"19a81cb5753781fff039fc75814c0f5c9b29436a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T10:15:04Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"5fd5b07bb6958a1fae528855a6c9efcbdb4f402c2e9b7032ac7f11deba4c2641","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034574367182032896","occurred_at":"2026-03-19T10:15:04Z","failed_at":"provision","signature_id":"6ba5e73bf0d6c568d592f93a828878de0942c26d453fe9571c0c6f10e14b5c43","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-19T10:34:51.543Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-19T10:34:54.096Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1706928777.bicepparam level=Error range=L20:32-L20:33 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T10:34:54.096Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1706928777.bicepparam level=Error range=L20:32-L20:33 code=BCP019\ntime=2026-03-19T10:34:54.097Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1706928777.bicepparam level=Error range=L21:31-L21:32 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T10:34:54.097Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1706928777.bicepparam level=Error range=L21:31-L21:32 code=BCP019\ntime=2026-03-19T10:34:54.097Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e","failure_pattern":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors","search_query":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034574270364913664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4503,"pr_state":"closed","pr_sha":"16b44df65e798f2ce166f58b4b3cdc45fae3ccec","final_merged_sha":"3a9b5c9cb4bcc81a28d2a98b1dd50a3c686f225a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T10:14:41Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"66e663f148e7bc2c78de69d1144524808d47ca3c0ad707ff2212eb6d165ec4fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034574270364913664","occurred_at":"2026-03-19T10:14:41Z","failed_at":"provision","signature_id":"db378bd4b50bd042c97d69b257bfc05c5fc5bf52aebaaae97b43a2345b1393cc","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-19T10:24:05.189Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-19T10:24:07.308Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1107424639.bicepparam level=Error range=L20:32-L20:33 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T10:24:07.309Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1107424639.bicepparam level=Error range=L20:32-L20:33 code=BCP019\ntime=2026-03-19T10:24:07.309Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1107424639.bicepparam level=Error range=L21:31-L21:32 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T10:24:07.309Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1107424639.bicepparam level=Error range=L21:31-L21:32 code=BCP019\ntime=2026-03-19T10:24:07.309Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e","failure_pattern":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors","search_query":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034573156340994048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4505,"pr_state":"closed","pr_sha":"9e4e5bd89728a904bc37ae0fc5c1633e841f6791","final_merged_sha":"ec8c17ac019d36b2be42061c5ce822f44607110b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T10:10:15Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"28ea254ac3ef1562ad4bbf4f2d59e2f34444741b657923cbd2d7d5b381913f72","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034573156340994048","occurred_at":"2026-03-19T10:10:15Z","failed_at":"provision","signature_id":"cda65ede66cf928b5f0decdf1e07e6558da562e8e26392c6db45626847556b67","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-19T10:33:04.131Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-19T10:33:06.290Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-3601094097.bicepparam level=Error range=L20:32-L20:33 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T10:33:06.290Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-3601094097.bicepparam level=Error range=L20:32-L20:33 code=BCP019\ntime=2026-03-19T10:33:06.290Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-3601094097.bicepparam level=Error range=L21:31-L21:32 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T10:33:06.290Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-3601094097.bicepparam level=Error range=L21:31-L21:32 code=BCP019\ntime=2026-03-19T10:33:06.290Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e","failure_pattern":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors","search_query":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034572639757930496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4529,"pr_state":"closed","pr_sha":"b1f9c4a6d0f13c973a84bb6ed80d5b39ab7e0773","final_merged_sha":"b1f9c4a6d0f13c973a84bb6ed80d5b39ab7e0773","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-19T10:08:12Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"9d9250b3679f59b5a9aa1dc94208d079ff4f72b02da1e118a4d15abc463cceda","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034572639757930496","occurred_at":"2026-03-19T10:08:12Z","failed_at":"provision","signature_id":"583536387c6e74b13ab20f071e52baeea9eaab90dd3297628f93b3c4355262ab","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-19T10:25:42.372Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-19T10:25:44.551Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-3146666533.bicepparam level=Error range=L20:32-L20:33 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T10:25:44.551Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-3146666533.bicepparam level=Error range=L20:32-L20:33 code=BCP019\ntime=2026-03-19T10:25:44.551Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-3146666533.bicepparam level=Error range=L21:31-L21:32 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T10:25:44.551Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-3146666533.bicepparam level=Error range=L21:31-L21:32 code=BCP019\ntime=2026-03-19T10:25:44.551Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e","failure_pattern":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors","search_query":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4533/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034572314405769216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4533,"pr_state":"closed","pr_sha":"3f3e9cbb83ba0accde3d0ddaa6f2b7e0150b5df2","final_merged_sha":"212f334cfaf06e962ef8ce0768aed92badda92c9","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T10:06:54Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"29b3e9d049e3ca0a52716fe8699c7962e9f49eb1837ea755c0495932cdac6bc7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4533/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034572314405769216","occurred_at":"2026-03-19T10:06:54Z","failed_at":"provision","signature_id":"a57870192bdce18165d9298b1a789f69935f01a6a9578befa2064526aa0331bd","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-19T10:23:22.006Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-19T10:23:24.390Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-2262083953.bicepparam level=Error range=L20:32-L20:33 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T10:23:24.390Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-2262083953.bicepparam level=Error range=L20:32-L20:33 code=BCP019\ntime=2026-03-19T10:23:24.390Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-2262083953.bicepparam level=Error range=L21:31-L21:32 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T10:23:24.390Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-2262083953.bicepparam level=Error range=L21:31-L21:32 code=BCP019\ntime=2026-03-19T10:23:24.390Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e","failure_pattern":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors","search_query":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034566183113461760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4503,"pr_state":"closed","pr_sha":"16b44df65e798f2ce166f58b4b3cdc45fae3ccec","final_merged_sha":"3a9b5c9cb4bcc81a28d2a98b1dd50a3c686f225a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T09:42:33Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"5d9c51fa21bf6a59b5e81250e2c27257a59ee6c650a15a58b81e5c81027c5d3b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034566183113461760","occurred_at":"2026-03-19T09:42:33Z","failed_at":"provision","signature_id":"82aff5975d73d1a783e97e3ba36c8c445d24ad58da288c0c524751d8c8d0a8c7","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-19T09:59:52.254Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-19T09:59:54.613Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-2667241276.bicepparam level=Error range=L20:32-L20:33 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T09:59:54.613Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-2667241276.bicepparam level=Error range=L20:32-L20:33 code=BCP019\ntime=2026-03-19T09:59:54.613Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-2667241276.bicepparam level=Error range=L21:31-L21:32 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T09:59:54.613Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-2667241276.bicepparam level=Error range=L21:31-L21:32 code=BCP019\ntime=2026-03-19T09:59:54.613Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e","failure_pattern":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors","search_query":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4514/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034564373132873728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4514,"pr_state":"closed","pr_sha":"61e704d766a95da5a14cf10c199fb79215449bb4","final_merged_sha":"61e704d766a95da5a14cf10c199fb79215449bb4","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-19T09:35:21Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"30447bb716c3faa8c2ed3078282206125662cf83035edf4e3fba394f62b1b61d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4514/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034564373132873728","occurred_at":"2026-03-19T09:35:21Z","failed_at":"provision","signature_id":"a04e5a64529690ca6c6fe7b9c4f6c450175ffaab7ac0e7d68a5fa55373a6d444","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-19T09:37:43.299Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-19T09:37:46.061Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1700176026.bicepparam level=Error range=L20:32-L20:33 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T09:37:46.061Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1700176026.bicepparam level=Error range=L20:32-L20:33 code=BCP019\ntime=2026-03-19T09:37:46.061Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1700176026.bicepparam level=Error range=L21:31-L21:32 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T09:37:46.062Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1700176026.bicepparam level=Error range=L21:31-L21:32 code=BCP019\ntime=2026-03-19T09:37:46.062Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e","failure_pattern":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors","search_query":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4532/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034563525027827712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4532,"pr_state":"closed","pr_sha":"1cc5acb519352221595acd3a1e7ec2eba6e27c40","final_merged_sha":"8033aacf4605ee2e3dd591bd5116c0e8c0a2b5f2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T09:31:55Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"db76c573d99961dccd6d9e89d076e60391482bb6f676a906eac140dc24d88021","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4532/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034563525027827712","occurred_at":"2026-03-19T09:31:55Z","failed_at":"provision","signature_id":"e41789cc65da383b0bb15852346bf3085b884d8ee63f412388e7ab02b74af1cf","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-19T09:49:35.382Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-19T09:49:38.018Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1162630994.bicepparam level=Error range=L20:32-L20:33 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T09:49:38.018Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1162630994.bicepparam level=Error range=L20:32-L20:33 code=BCP019\ntime=2026-03-19T09:49:38.018Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1162630994.bicepparam level=Error range=L21:31-L21:32 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T09:49:38.018Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1162630994.bicepparam level=Error range=L21:31-L21:32 code=BCP019\ntime=2026-03-19T09:49:38.018Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e","failure_pattern":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors","search_query":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4517/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034563017525432320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4517,"pr_state":"closed","pr_sha":"6d35b73d52257ae7a5bbc02428187d621caa6d9a","final_merged_sha":"98f9b5f64bc2a1090552d4cecf1ed332bfca4775","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T09:29:58Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"6b3ce36fc1f86f004cf6942bbe50b72a108edbf2c5701b3bf13564f338928098","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4517/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034563017525432320","occurred_at":"2026-03-19T09:29:58Z","failed_at":"provision","signature_id":"e4b6536a21f0e6d64f1a66718758731e97527facf32efc29058ddfb2ae0a7ee7","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-19T09:47:01.978Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-19T09:47:06.249Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-3597627977.bicepparam level=Error range=L20:32-L20:33 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T09:47:06.249Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-3597627977.bicepparam level=Error range=L20:32-L20:33 code=BCP019\ntime=2026-03-19T09:47:06.250Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-3597627977.bicepparam level=Error range=L21:31-L21:32 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T09:47:06.250Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-3597627977.bicepparam level=Error range=L21:31-L21:32 code=BCP019\ntime=2026-03-19T09:47:06.250Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e","failure_pattern":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors","search_query":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4514/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034552139140829184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4514,"pr_state":"closed","pr_sha":"61e704d766a95da5a14cf10c199fb79215449bb4","final_merged_sha":"61e704d766a95da5a14cf10c199fb79215449bb4","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-19T08:46:44Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"a5a53d2e7549bf34c97bb5f70533a57f1d79efa1570d5a78c40f2d17bbe044aa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4514/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034552139140829184","occurred_at":"2026-03-19T08:46:44Z","failed_at":"provision","signature_id":"093ba92dc476c671cab0c24530d0b0983c6e1378668b3fcb3c269acba72a10c4","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-19T09:03:32.103Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-19T09:03:34.838Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-2996185070.bicepparam level=Error range=L20:32-L20:33 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T09:03:34.838Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-2996185070.bicepparam level=Error range=L20:32-L20:33 code=BCP019\ntime=2026-03-19T09:03:34.838Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-2996185070.bicepparam level=Error range=L21:31-L21:32 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T09:03:34.838Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-2996185070.bicepparam level=Error range=L21:31-L21:32 code=BCP019\ntime=2026-03-19T09:03:34.838Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e","failure_pattern":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors","search_query":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4531/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034551843652112384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4531,"pr_state":"closed","pr_sha":"9a1ec2ad62b9e33d2818fcc6ddb45cc8f116ad6d","final_merged_sha":"e3f0945f33d4cde7c0df12f230b387f72c667e3f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T08:45:34Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ccd2aa51f3e58c4f1bd70535a20f379e0176759d6e65e41929cab8fd01768ef5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4531/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034551843652112384","occurred_at":"2026-03-19T08:45:34Z","failed_at":"provision","signature_id":"149ed53799cae3aadf8f58c62574d05d83bd9e482279c10c03798592e6487e92","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-19T09:07:31.365Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-19T09:07:34.273Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1780113172.bicepparam level=Error range=L20:32-L20:33 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T09:07:34.273Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1780113172.bicepparam level=Error range=L20:32-L20:33 code=BCP019\ntime=2026-03-19T09:07:34.273Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1780113172.bicepparam level=Error range=L21:31-L21:32 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T09:07:34.273Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1780113172.bicepparam level=Error range=L21:31-L21:32 code=BCP019\ntime=2026-03-19T09:07:34.273Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e","failure_pattern":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors","search_query":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4481/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034549319230558208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4481,"pr_state":"closed","pr_sha":"a0ece52bde4e4930997796ae17d728b08f758b6b","final_merged_sha":"a0ece52bde4e4930997796ae17d728b08f758b6b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-19T08:35:32Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"8260e85bbb3ae9d7a57be468facc722fd0f736a7bf95d290eff052b9b3f7c2a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4481/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034549319230558208","occurred_at":"2026-03-19T08:35:32Z","failed_at":"provision","signature_id":"9a4a1e3196c26f0159d84d3b31e6302f5d912daad23d07386aef2693ad02070b","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-19T08:59:26.011Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-19T08:59:28.769Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-3535488828.bicepparam level=Error range=L20:32-L20:33 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T08:59:28.769Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-3535488828.bicepparam level=Error range=L20:32-L20:33 code=BCP019\ntime=2026-03-19T08:59:28.769Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-3535488828.bicepparam level=Error range=L21:31-L21:32 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T08:59:28.770Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-3535488828.bicepparam level=Error range=L21:31-L21:32 code=BCP019\ntime=2026-03-19T08:59:28.770Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e","failure_pattern":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors","search_query":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4517/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034549121745948672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4517,"pr_state":"closed","pr_sha":"6d35b73d52257ae7a5bbc02428187d621caa6d9a","final_merged_sha":"98f9b5f64bc2a1090552d4cecf1ed332bfca4775","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T08:34:45Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"8aee26c7901111a775114795fec7199d7a45b30f8d96dd816c73cda4aeeeb110","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4517/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034549121745948672","occurred_at":"2026-03-19T08:34:45Z","failed_at":"provision","signature_id":"06a38e6096d1165598afd624bb9ba084b3b2af23480511afffbb4ea34449cecd","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-19T08:51:18.262Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-19T08:51:20.544Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1030451609.bicepparam level=Error range=L20:32-L20:33 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T08:51:20.544Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1030451609.bicepparam level=Error range=L20:32-L20:33 code=BCP019\ntime=2026-03-19T08:51:20.544Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1030451609.bicepparam level=Error range=L21:31-L21:32 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-19T08:51:20.544Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-1030451609.bicepparam level=Error range=L21:31-L21:32 code=BCP019\ntime=2026-03-19T08:51:20.544Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e","failure_pattern":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors","search_query":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034547471283130368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4347,"pr_state":"closed","pr_sha":"a82c62541bf569a06efb0a6211c6ae9e8a2c01a4","final_merged_sha":"a82c62541bf569a06efb0a6211c6ae9e8a2c01a4","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-19T08:28:11Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034542325488685056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4529,"pr_state":"closed","pr_sha":"b1f9c4a6d0f13c973a84bb6ed80d5b39ab7e0773","final_merged_sha":"b1f9c4a6d0f13c973a84bb6ed80d5b39ab7e0773","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-19T08:07:44Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034538712666214400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4444,"pr_state":"closed","pr_sha":"092800808fd635f2494b3d3e1993603df989962b","final_merged_sha":"092800808fd635f2494b3d3e1993603df989962b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-19T07:53:23Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"95c1a25dfe65f38b2c9bd41d3a0d3eb8696aca8e1ff1af5b8c4899c33eb3d599","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034538712666214400","occurred_at":"2026-03-19T07:53:23Z","failed_at":"provision","signature_id":"0002355679e8107993d7c3e66c77544304cf6fc52431a67be03a7030071f7fd4","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-19T08:32:05.480Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-19T08:32:06.352Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:b3bd168a-18e0-4889-b4dd-953e3530d85b cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j6214400] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j6214400.hcpsvc.osadev.cloud] image:map[digest:sha256:796f9735ba4abe15568fe2c3d1460a6e33c0f57d0a05c556575de899c7724da1 registry:arohcpsvcdev.azurecr.io repository:ci-op-3fx1iyfr/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-3fx1iyfr/pipeline:pull workloadIdentityClientId:3d742a6c-79ed-4a6a-a366-734a1879a845 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:e5b3c84d-0aba-4357-b0df-7fbd92b15003 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-03-19T08:32:06.352Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-19T08:32:06.525Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy namespace=sessiongate\ntime=2026-03-19T08:32:06.777Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy dryRun=false\ntime=2026-03-19T08:32:06.777Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-19T08:32:06.883Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-19T08:37:14.100Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"resource not ready, name: sessiongate, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-19T08:37:14.100Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-19T08:37:14.100Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-19T08:37:17.958Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy release=sessiongate namespace=sessiongate status=failed description=\"Release \\\"sessiongate\\\" failed: resource not ready, name: sessiongate, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:b3bd168a-18e0-4889-b4dd-953e3530d85b cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j6214400] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j6214400.hcpsvc.osadev.cloud] image:map[digest:sha256:796f9735ba4abe15568fe2c3d1460a6e33c0f57d0a05c556575de899c7724da1 registry:arohcpsvcdev.azurecr.io repository:ci-op-3fx1iyfr/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-3fx1iyfr/pipeline:pull workloadIdentityClientId:3d742a6c-79ed-4a6a-a366-734a1879a845 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:e5b3c84d-0aba-4357-b0df-7fbd92b15003 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-03-19T08:37:17.960Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5SUzW7bOhCF93oKQqsYEAUnuchtXWSRnwIFChSGm1WKLihqIjGhOOrMMEGCPHyhyK4FVbbTnX10+M2Qh5zEgygCxkgWWJ2r0ogRU3g4wuIerHx1oVywkAtVpnrpm2ngjwSPEN4Ubo3dyLMfiVJKpSv4FYHlIkoNQZw14jCkmUoZmB2Gygho6k3aRKlH39Ks53wHSyBLwkdXAl15wzymsG07yTywdiwOtQsVQefbIOjRWbiwFmOQv+pMlr3ykQVohX7s0OvfujHBVNBAj9wsvIbW4/NGPVxop19bIHF33cl1B3VHwDXQ3o1eWFpG7y9dKF2oOmsbvdfF9v9EA11CSO7lLaAlemef34p4j0/jMv8OaEDIWdZ8+CT2cgJKvXsPX25uliuME+jpcCMLNqv11b+GOxfc6HpyPliXG0Jd2zY3L5Egt9gME5+6IxaDEHrfBzbRwv5F2oMpgTR4sBPvZgi5nAx3yNof//sYo34OMNcP7kAYyc9PyUMsgAII8OdumHDyqp5qIFDiGmAxTasKkCeAoI5KI9DJR+nJ/ORMz0/18ceb+YfF6clifnabzlSeq52e/xfH/92ms1nyqu7RBfXgQnnuQgAajEAMajv5hiNvPOuSV9USdl+3jWaqxbI3vw+SKY9VphpgNlWHRCqBVPE82HwJbH8HAAD//5q3HKOoBQAA\"\ntime=2026-03-19T08:37:17.961Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-19T08:37:17.962Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: sessiongate, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3796d4e6741d5657aa106b46b84d6ee7c621cc8a323332361ce1d435c3813f80","failure_pattern":"resource not ready, name: sessiongate, kind: Deployment, status: InProgress context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: sessiongate, kind: Deployment, status: InProgress\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["3796d4e6741d5657aa106b46b84d6ee7c621cc8a323332361ce1d435c3813f80"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4528/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034538526338453504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4528,"pr_state":"closed","pr_sha":"671f1cceb6240ffe9b282b20bd18e2d88f2f85a2","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-19T07:52:39Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4517/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034517510669209600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4517,"pr_state":"closed","pr_sha":"6d35b73d52257ae7a5bbc02428187d621caa6d9a","final_merged_sha":"98f9b5f64bc2a1090552d4cecf1ed332bfca4775","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-19T06:29:08Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034481813438599168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4497,"pr_state":"closed","pr_sha":"ef00dbb4f2576014164324de3817c70002a260cc","final_merged_sha":"ef00dbb4f2576014164324de3817c70002a260cc","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-19T04:07:17Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4396/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034479296470323200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4396,"pr_state":"closed","pr_sha":"732c1753890fbef90e84268733f7e4d18f32e691","final_merged_sha":"732c1753890fbef90e84268733f7e4d18f32e691","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-19T03:57:12Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4425/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034460478557655040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4425,"pr_state":"closed","pr_sha":"64a2f7a8b65b208052e3c620a9c0db2c86c6a372","final_merged_sha":"855eea85f3052237fc8528c6d280c0ac41ee4307","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-19T02:42:31Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034447702539898880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4045,"pr_state":"open","pr_sha":"3207545fca598a0bbc926207da9f3ebd12d86873","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T01:51:45Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"43d9823a6d5105305421ab566ee5190f3294b17ee8db6e8d5b990e7a11fa8eca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034447702539898880","occurred_at":"2026-03-19T01:51:45Z","failed_at":"e2e","signature_id":"000de2ec458ae4b10e9e67e92d2b62c284878229804652f0c59181113be37160","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:281]: Timed out after 30.000s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000fe61e0\u003e: \n read access verification failed: get pods -n kube-system: pods is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"pods\" in API group \"\" in the namespace \"kube-system\"\n get configmaps -n kube-system: configmaps is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"configmaps\" in API group \"\" in the namespace \"kube-system\"\n {\n msg: \"read access verification failed: get pods -n kube-system: pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\\nget configmaps -n kube-system: configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.joinError | 0xc000d315a8\u003e{\n errs: [\n \u003c*fmt.wrapError | 0xc000fc5e80\u003e{\n msg: \"get pods -n kube-system: pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.StatusError | 0xc000fd8be0\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n Reason: \"Forbidden\",\n Details: {Name: \"\", Group: \"\", Kind: \"pods\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n },\n },\n \u003c*fmt.wrapError | 0xc000fe6180\u003e{\n msg: \"get configmaps -n kube-system: configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.StatusError | 0xc000fd8e60\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n Reason: \"Forbidden\",\n Details: {Name: \"\", Group: \"\", Kind: \"configmaps\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n },\n },\n ],\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:281]: Timed out after 30.000s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000fe61e0\u003e: \n read access verification failed: get pods -n kube-system: pods is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"pods\" in API group \"\" in the namespace \"kube-system\"\n get configmaps -n kube-system: configmaps is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"configmaps\" in API group \"\" in the namespace \"kube-system\"\n {\n msg: \"read access verification failed: get pods -n kube-system: pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\\nget configmaps -n kube-system: configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.joinError | 0xc000d315a8\u003e{\n errs: [\n \u003c*fmt.wrapError | 0xc000fc5e80\u003e{\n msg: \"get pods -n kube-system: pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.StatusError | 0xc000fd8be0\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n Reason: \"Forbidden\",\n Details: {Name: \"\", Group: \"\", Kind: \"pods\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n },\n },\n \u003c*fmt.wrapError | 0xc000fe6180\u003e{\n msg: \"get configmaps -n kube-system: configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.StatusError | 0xc000fd8e60\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n Reason: \"Forbidden\",\n Details: {Name: \"\", Group: \"\", Kind: \"configmaps\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n },\n },\n ],\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3c6a69913a4e7e3ddde30c4ad0c3578c4319954e53163360fc7432ced31dda4a","failure_pattern":"read access verification failed: get pod \u003cpod\u003e kube-system: pods is forbidden: User \"system:sre-break-glass:\u003cuuid\u003e\" cannot list resource \"pods\" in API group \"\" in the namespace \"\u003cnamespace\u003e\"","search_query":"read access verification failed: get pods -n kube-system: pods is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"pods\" in API group \"\" in the namespace \"kube-system\""}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["3c6a69913a4e7e3ddde30c4ad0c3578c4319954e53163360fc7432ced31dda4a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4095/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034447633824616448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4095,"pr_state":"open","pr_sha":"8f711fab81a8d9300197820c371560b67cc2acd0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-19T01:51:28Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4396/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034446238568419328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4396,"pr_state":"closed","pr_sha":"732c1753890fbef90e84268733f7e4d18f32e691","final_merged_sha":"732c1753890fbef90e84268733f7e4d18f32e691","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-19T01:45:55Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034445755703365632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4444,"pr_state":"closed","pr_sha":"092800808fd635f2494b3d3e1993603df989962b","final_merged_sha":"092800808fd635f2494b3d3e1993603df989962b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-19T01:44:00Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"13f1a93b6dc5892623bdb5aa534a4f91640aac50148af68038988bfc824df8bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034445755703365632","occurred_at":"2026-03-19T01:44:00Z","failed_at":"e2e","signature_id":"d66cf14e9038065d1acec11111844f9e22a6b1f40fb14378f8974f01c13088cf","test_name":"Engineering should be able to retrieve expected metrics from the /metrics endpoint","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:74]: Failed to set up port-forward\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0000fd940\u003e: \n port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error\n {\n msg: \"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error\",\n err: \u003c*errors.errorString | 0xc000812ad0\u003e{\n s: \"error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:74]: Failed to set up port-forward\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0000fd940\u003e: \n port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error\n {\n msg: \"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error\",\n err: \u003c*errors.errorString | 0xc000812ad0\u003e{\n s: \"error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error\",\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f27efe24b36721c3037189b0270b3c037f3dcd8e282db885731d7395d97e1e2a","failure_pattern":"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing \u003cip\u003e:\u003cport\u003e, code 500: 500 Internal Server Error","search_query":"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f27efe24b36721c3037189b0270b3c037f3dcd8e282db885731d7395d97e1e2a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4517/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034426396922613760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4517,"pr_state":"closed","pr_sha":"395a868c3f8c57cce322420a4a737fc37987f57e","final_merged_sha":"98f9b5f64bc2a1090552d4cecf1ed332bfca4775","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-19T00:27:05Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"10b009f38255b150ab35cb0351b47fe8983ddecde15f72802f1ba096cf1840ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4517/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034426396922613760","occurred_at":"2026-03-19T00:27:05Z","failed_at":"provision","signature_id":"c027c4d12f6a3d02006d0282f3d5b1020328ed77f7bf43d76c7a5e7566e656ac","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-03-19T00:50:22.744Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-03-19T00:50:24.500Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-03-19T00:50:26.505Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=a5cab0b34a14fa55e3bc4dfe4c6c647490c0b66909a96c4f8622360e3cf9c5e4\ntime=2026-03-19T01:04:00.012Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j2613760-svc/providers/Microsoft.Resources/deployments/a5cab0b34a14fa55e3bc4dfe4c6c647490c0b66909a96c4f8622360e3cf9c5e4/operationStatuses/08584277254600459045\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j2613760-svc/providers/Microsoft.Resources/deployments/profile-kaxx74lugt6jc\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j2613760-svc/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j2613760-svc/resourceAssociations/nsp-j2613760-svc-ra-34e5f7fa-afce-5359-977c-a16f66dca6ad\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"HttpRequestException\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Error while processing request for association /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j2613760-svc/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j2613760-svc/resourceAssociations/nsp-j2613760-svc-ra-34e5f7fa-afce-5359-977c-a16f66dca6ad, Error message: An error occurred while sending the request.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9220c54af3cdcf82280969ee7ca39e91829e564fcd5ac622742ec149542a29fd","failure_pattern":"ERROR CODE: DeploymentFailed; detail code HttpRequestException; detail message Error while processing network security perimeter association: An error occurred while sending the request.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9220c54af3cdcf82280969ee7ca39e91829e564fcd5ac622742ec149542a29fd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034422787019378688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4473,"pr_state":"closed","pr_sha":"e5513c653f9bc0042d7f813015e5db3573298a74","final_merged_sha":"8b7576c81d01104dc238546ec2678952bf7afd61","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-19T00:12:43Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}