{"meta":{"date":"2026-03-20","timezone":"UTC","generated_at":"2026-10-31T23:16:07Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":66,"failed_runs":24,"runs_with_occurrences":24,"runs_with_failure_pattern_matches":21,"runs_with_unmatched_occurrences":3,"runs_without_occurrence_rows":42,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4561/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035136649410646016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4561,"pr_state":"closed","pr_sha":"15ae132308931cc541a77791ad2f97ce6969b797","final_merged_sha":"15ae132308931cc541a77791ad2f97ce6969b797","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-20T23:29:22Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"5aba4f5636e5056e4e78009cc0cb87c9a44f58810478fffa558b2a4b291c536e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4561/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035136649410646016","occurred_at":"2026-03-20T23:29:22Z","failed_at":"provision","signature_id":"c027c4d12f6a3d02006d0282f3d5b1020328ed77f7bf43d76c7a5e7566e656ac","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-03-20T23:48:44.111Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-03-20T23:48:45.904Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-03-20T23:48:47.479Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=ca137337b764e3c08374540c6857f76d29e70e48046358dee986e5aa349b5ed0\ntime=2026-03-21T00:01:20.845Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j0646016-svc/providers/Microsoft.Resources/deployments/ca137337b764e3c08374540c6857f76d29e70e48046358dee986e5aa349b5ed0/operationStatuses/08584275563588572205\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/admin-api-cert-6yq2qgn2eojyi\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deploymentScripts/newCert-admin-api-cert-prow-j0646016\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentScriptOperationFailed\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The request timed out. The tracking activity id is '3883f919-dadd-4ac6-9670-ce69949b6d5b', the correlation id is 'd15f5c50-8037-4187-a76c-bbf8da08ebb2'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"72beca7d803d4cedb56326009ac9bab7932b11b77fb407c9d9dbabb435e66c1c","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentScriptOperationFailed; detail message The request timed out. The tracking activity id is '\u003cuuid\u003e', the correlation id is '\u003cuuid\u003e'.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["72beca7d803d4cedb56326009ac9bab7932b11b77fb407c9d9dbabb435e66c1c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035134834765991936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4505,"pr_state":"closed","pr_sha":"99239bedefa6d1a0ead71a0f556537628bee2928","final_merged_sha":"ec8c17ac019d36b2be42061c5ce822f44607110b","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-20T23:22:10Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4564/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035116738869202944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4564,"pr_state":"open","pr_sha":"5ff751ab4b65e7a493cd14a11af7233b97d10f0c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-20T22:10:15Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035096867066351616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4562,"pr_state":"closed","pr_sha":"b6b37cf1835fe3b9bf2d1f7d7446f48e9884ae59","final_merged_sha":"b6b37cf1835fe3b9bf2d1f7d7446f48e9884ae59","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T20:51:17Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4493/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035081554203512832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4493,"pr_state":"closed","pr_sha":"8a3e2673bc025c3e97823e6867d3b9ede213d29b","final_merged_sha":"8a3e2673bc025c3e97823e6867d3b9ede213d29b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T19:50:25Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035079358124658688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4477,"pr_state":"closed","pr_sha":"74984b22beb987028d79f91afae029e9cae45caa","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-20T19:41:42Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d989cdf37b146b72545f75f77e67a771b6f0262bea129957d99cd1d8e439a346","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035079358124658688","occurred_at":"2026-03-20T19:41:42Z","failed_at":"e2e","signature_id":"a46b35450a1a2ea1b98ae235edd6dc95d844c1ee1154a6251578b54628d54d64","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0013b25a0\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-tbqvdk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-tbqvdk, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-tbqvdk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-tbqvdk, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013eb8c0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-tbqvdk\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0013b25a0\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-tbqvdk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-tbqvdk, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-tbqvdk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-tbqvdk, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013eb8c0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-tbqvdk\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-tbqvdk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-tbqvdk, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4560/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035077533304623104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4560,"pr_state":"closed","pr_sha":"8f5945dac324330eef5fadbd1695f56af8b2406f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-20T19:34:28Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"b98cf25ea92d0bd0a7d58872b63c8c739527c281ad3bc87d5699fba4b4b42b38","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4560/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035077533304623104","occurred_at":"2026-03-20T19:34:28Z","failed_at":"provision","signature_id":"cfca688c633af28c54b31f65ec656c5ca079eafea484e6d884acc4d2ddae0ece","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-velero-image","test_suite":"step graph","failure_text":"time=2026-03-20T20:18:41.321Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image description=\"Step mirror-velero-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e velero.server.registry \u003cnil\u003e}:{\u003cnil\u003e velero.server.repository \u003cnil\u003e}@{\u003cnil\u003e velero.server.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-20T20:19:14.034Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org\\nFetch pull secret for source registry registry.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nFailed to log in to ACR arohcpsvcdev: ERROR: Could not connect to the registry login server 'arohcpsvcdev.azurecr.io'. Please verify that the registry exists and the URL 'https://arohcpsvcdev.azurecr.io/v2/' is reachable from your environment.\\nTry running 'az acr check-health -n arohcpsvcdev --yes' to diagnose this issue.\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"061908f35a0fa01699ae0bafb53b5a8e0ffd0428c6990d52950e0f0f82a49466","failure_pattern":"Failed to log in to ACR arohcpsvcdev: ERROR: Could not connect to the registry login server 'arohcpsvcdev.azurecr.io'. Please verify that the registry exists and the URL '\u003curl\u003e' is reachable from your environment.","search_query":"Failed to log in to ACR arohcpsvcdev: ERROR: Could not connect to the registry login server 'arohcpsvcdev.azurecr.io'. Please verify that the registry exists and the URL 'https://arohcpsvcdev.azurecr.io/v2/' is reachable from your environment."}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["061908f35a0fa01699ae0bafb53b5a8e0ffd0428c6990d52950e0f0f82a49466"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4560/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035069808260681728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4560,"pr_state":"closed","pr_sha":"7bc9a51f7b97e79fc6aff907893866eed52babe1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-20T19:03:46Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"855993e244fa0901a4ae141f6e4e10d020918e3d8c5de938255391c3b168fe2c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4560/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035069808260681728","occurred_at":"2026-03-20T19:03:46Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035067618099990528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4457,"pr_state":"open","pr_sha":"e713e6d1f13e575ac5726c6bdbf7cf4e94645db1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-20T18:55:04Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035063795235164160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4562,"pr_state":"closed","pr_sha":"b6b37cf1835fe3b9bf2d1f7d7446f48e9884ae59","final_merged_sha":"b6b37cf1835fe3b9bf2d1f7d7446f48e9884ae59","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T18:39:53Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4559/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035059680077680640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4559,"pr_state":"closed","pr_sha":"88537065778ffab09a3fb724a338fca3bda8a0d9","final_merged_sha":"88537065778ffab09a3fb724a338fca3bda8a0d9","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T18:23:31Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035057833598259200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4473,"pr_state":"closed","pr_sha":"cb025a43241b8f8a4aab0db7e2f132e10efed210","final_merged_sha":"8b7576c81d01104dc238546ec2678952bf7afd61","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-20T18:16:11Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"cfbe7a2ae00991df72c4ce9da59a3f029006628574deedaab11ab93087567a5d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035057833598259200","occurred_at":"2026-03-20T18:16:11Z","failed_at":"e2e","signature_id":"49e97d8d69f5932ebdaa11f6f8d5ba824becacf9c00cc69398ebdd238fb7d6c6","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:119]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0015d4c20\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-872h6l\": PUT https://rp.j8259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-872h6l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-872h6l\\\": PUT https://rp.j8259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-872h6l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value: field cannot be null\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0015d4c00\u003e{\n msg: \"failed starting cluster creation \\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-872h6l\\\": PUT https://rp.j8259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-872h6l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value: field cannot be null\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0015f5920\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Request-Id\": [\n \"c1b4f007-c6e2-4269-baf5-c0064e14ae5e\",\n ],\n \"Date\": [\n \"Fri, 20 Mar 2026 19:01:43 GMT\",\n ],\n \"Content-Length\": [\"206\"],\n \"X-Envoy-Upstream-Service-Time\": [\"39\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0015d4b40\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 82, 101, 113, 117, 105, 114, 101, 100, 32, 118, 97, 108, 117, 101, 58, 32, 102, 105, 101, 108, 100, 32, 99, 97, 110, 110, 111, 116, 32, 98, 101, 32, 110, 117, 108, 108, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 112, 114, 111, 112, 101, 114, 116, 105, 101, 115, 46, 101, 116, 99, 100, 46, 100, 97, 116, 97, 69, 110, 99, 114, 121, 112, 116, 105, 111, 110, 46, 99, 117, 115, 116, 111, 109, 101, 114, 77, 97, 110, 97, 103, 101, 100, 46, 107, 109, 115, 46, 118, 105, 115, 105, 98, 105, 108, 105, 116, 121, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 206,\n TransferEncoding: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:119]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0015d4c20\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-872h6l\": PUT https://rp.j8259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-872h6l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-872h6l\\\": PUT https://rp.j8259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-872h6l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value: field cannot be null\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0015d4c00\u003e{\n msg: \"failed starting cluster creation \\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-872h6l\\\": PUT https://rp.j8259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-872h6l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value: field cannot be null\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0015f5920\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Request-Id\": [\n \"c1b4f007-c6e2-4269-baf5-c0064e14ae5e\",\n ],\n \"Date\": [\n \"Fri, 20 Mar 2026 19:01:43 GMT\",\n ],\n \"Content-Length\": [\"206\"],\n \"X-Envoy-Upstream-Service-Time\": [\"39\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0015d4b40\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 82, 101, 113, 117, 105, 114, 101, 100, 32, 118, 97, 108, 117, 101, 58, 32, 102, 105, 101, 108, 100, 32, 99, 97, 110, 110, 111, 116, 32, 98, 101, 32, 110, 117, 108, 108, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 112, 114, 111, 112, 101, 114, 116, 105, 101, 115, 46, 101, 116, 99, 100, 46, 100, 97, 116, 97, 69, 110, 99, 114, 121, 112, 116, 105, 111, 110, 46, 99, 117, 115, 116, 111, 109, 101, 114, 77, 97, 110, 97, 103, 101, 100, 46, 107, 109, 115, 46, 118, 105, 115, 105, 98, 105, 108, 105, 116, 121, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 206,\n TransferEncoding: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1b131b2f9dd20d650a275cc9a5f957034289c13e33a1a1ca48259b4fd6d16bed","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1b131b2f9dd20d650a275cc9a5f957034289c13e33a1a1ca48259b4fd6d16bed"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035055835494748160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4505,"pr_state":"closed","pr_sha":"d63d1c51444694ea7519aa6e768b4440461a4ad2","final_merged_sha":"ec8c17ac019d36b2be42061c5ce822f44607110b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-20T18:08:15Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"08b4e37f022e86734eb7e2585db62ef3ac30b538390ad0d65078bf5b9b88f7ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035055835494748160","occurred_at":"2026-03-20T18:08:15Z","failed_at":"e2e","signature_id":"66c21a8a02d4b13f9f0e211adbd7cde8c5437cf7d345a4131d1722787b2a727a","test_name":"Customer should upgrade a nodepool from 4.20 to 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_minor_upgrade.go:183]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0009e0190\u003e: \n no nodes found in node pool \"npminorupgrade\"\n {\n s: \"no nodes found in node pool \\\"npminorupgrade\\\"\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_minor_upgrade.go:183]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0009e0190\u003e: \n no nodes found in node pool \"npminorupgrade\"\n {\n s: \"no nodes found in node pool \\\"npminorupgrade\\\"\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a2ac86b0ab1e7ebf67537eab9a4c4d1709247133039cee85710a8ee70ed3aada","failure_pattern":"no nodes found in node pool \"\u003cnodepool\u003e\"","search_query":"no nodes found in node pool \"npminorupgrade\""}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a2ac86b0ab1e7ebf67537eab9a4c4d1709247133039cee85710a8ee70ed3aada"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035051381378781184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4453,"pr_state":"closed","pr_sha":"74066469c7ba25be3efcb17e4e6a47885ab78793","final_merged_sha":"74066469c7ba25be3efcb17e4e6a47885ab78793","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T17:50:33Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4257/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035031283490361344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4257,"pr_state":"closed","pr_sha":"06e9152e575fe1af81ab23347da2f44373ebcf48","final_merged_sha":"f9b0dd45c8c2499c52d47ec79f8297cdae4dfe1f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-20T16:30:41Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"9d135b91afcb68b27d5eecc8d298b43af0d66c8a6cb536b6175f2dfd45225054","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4257/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035031283490361344","occurred_at":"2026-03-20T16:30:41Z","failed_at":"provision","signature_id":"0cfdbc69e36cd9409140a1d94f3eca54c19e2976278c5f5c45b1651fb5c59fcf","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster","test_suite":"step graph","failure_text":"time=2026-03-20T16:55:00.908Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-03-20T16:55:03.112Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster\ntime=2026-03-20T16:55:04.440Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster deployment=fb4a36dd0ff494fa35a9974bc8b6e8e5f0d46d48f18da1e495a99fed8805b844\ntime=2026-03-20T16:58:35.372Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j0361344-mgmt-1/providers/Microsoft.Resources/deployments/fb4a36dd0ff494fa35a9974bc8b6e8e5f0d46d48f18da1e495a99fed8805b844/operationStatuses/08584275811817549292\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0361344-mgmt-1/providers/Microsoft.Resources/deployments/cluster-wtgam6mzv3hok\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0361344-mgmt-1/providers/Microsoft.Resources/deployments/prow-j0361344-mgmt-1-outbound-ip\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0361344-mgmt-1/providers/Microsoft.Resources/deployments/prow-j0361344-mgmt-1-outbound-ip\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035030283119497216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4457,"pr_state":"open","pr_sha":"e713e6d1f13e575ac5726c6bdbf7cf4e94645db1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-20T16:26:37Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"53d44ef420b26a03b75afe5c3e2013acbce58980c6280cf209cbb90bd48c99a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035030283119497216","occurred_at":"2026-03-20T16:26:37Z","failed_at":"provision","signature_id":"0cfdbc69e36cd9409140a1d94f3eca54c19e2976278c5f5c45b1651fb5c59fcf","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster","test_suite":"step graph","failure_text":"time=2026-03-20T16:54:10.821Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-03-20T16:54:11.879Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster\ntime=2026-03-20T16:54:13.398Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster deployment=0d7e6aa49867da5a1f1c056487b5b629b56beeebeef7c241f5ff48c4143e8581\ntime=2026-03-20T16:57:44.411Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9497216-mgmt-1/providers/Microsoft.Resources/deployments/0d7e6aa49867da5a1f1c056487b5b629b56beeebeef7c241f5ff48c4143e8581/operationStatuses/08584275812329836622\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9497216-mgmt-1/providers/Microsoft.Resources/deployments/cluster-nvvsssnj7yrsk\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9497216-mgmt-1/providers/Microsoft.ContainerService/managedClusters/prow-j9497216-mgmt-1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Azure has a fixed 4000 role assignments limit per subscription. Please identify and delete useless role assingments before proceeding the operation. See https://learn.microsoft.com/en-us/azure/role-based-access-control/troubleshoot-limits for more details. Create role assignment failed. Scope: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9497216-mgmt-1-aks1; role assignment name: c7ba168c-98d3-4cc7-8bc5-d68c81ff6899. No more role assignments can be created.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d19aa52574cc822a26d2c3b6b5475e049a3272426fea3b2aedbec1e04e333297","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d19aa52574cc822a26d2c3b6b5475e049a3272426fea3b2aedbec1e04e333297"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4510/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035029279707762688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4510,"pr_state":"closed","pr_sha":"704b8cdeaa0c942e9c4bedf387c789d55bbec4cd","final_merged_sha":"a3c49a02699147584f90d5a78315b81d0226ff8a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-20T16:22:43Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"cab38afdd563e742d63fc8692230b3981207a8345467c694236e0f919867899e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4510/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035029279707762688","occurred_at":"2026-03-20T16:22:43Z","failed_at":"provision","signature_id":"0cfdbc69e36cd9409140a1d94f3eca54c19e2976278c5f5c45b1651fb5c59fcf","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster","test_suite":"step graph","failure_text":"time=2026-03-20T16:45:51.997Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-03-20T16:45:52.845Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster\ntime=2026-03-20T16:45:54.113Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster deployment=9d866b4507ed610ff4cd012d65ac8a88c2797ef7dec33752bfeb6b0d5ca19bcf\ntime=2026-03-20T16:55:56.289Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j7762688-mgmt-1/providers/Microsoft.Resources/deployments/9d866b4507ed610ff4cd012d65ac8a88c2797ef7dec33752bfeb6b0d5ca19bcf/operationStatuses/08584275817320700767\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7762688-mgmt-1/providers/Microsoft.Resources/deployments/metrics-infra\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7762688-mgmt-1/providers/Microsoft.Resources/deployments/1bfc0c3d-0803-54a4-98d0-9dd681a84c55\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7762688-mgmt-1/providers/Microsoft.Resources/deployments/e87ba3a5-5c8b-591c-ae06-77192812ed2b\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7762688-mgmt-1/providers/Microsoft.Resources/deployments/e8f77804-fe51-5e05-99f0-c3857de6fcd9\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7762688-mgmt-1/providers/Microsoft.Resources/deployments/6417f1c9-492b-5383-89d9-d3a268821cc8\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7762688-mgmt-1/providers/Microsoft.Resources/deployments/359981df-cf25-5199-bb86-db1b6021d9b5\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7762688-mgmt-1/providers/Microsoft.Resources/deployments/85f0b68f-9165-53f4-bad1-ddf2f1eaa33c\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035028131919695872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4477,"pr_state":"closed","pr_sha":"280a9cf2b7882121824a6f22734cf7495ec61e04","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-20T16:18:03Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"73632b6dbe04f3f8e397655a9517fd8134c3069a3403338523f7dd5059d1809b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035028131919695872","occurred_at":"2026-03-20T16:18:03Z","failed_at":"provision","signature_id":"eb42f536566c06969e79bb4dd530b90fade97f988fe3d18503444c2160414c7b","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/svc-mgmt-permissions","test_suite":"step graph","failure_text":"time=2026-03-20T16:55:44.508Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions description=\"Step svc-mgmt-permissions\\n Kind: ARM\\n Template: templates/svc-mgmt-permissions.bicep\\n Parameters: configurations/svc-mgmt-permissions.tmpl.bicepparam\"\ntime=2026-03-20T16:55:45.057Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions\ntime=2026-03-20T16:55:46.166Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions deployment=01e82a9dbd393631d4cdc403e6d2a13d0a5954cb6aa84fb7f837e5e0a6b958ce\ntime=2026-03-20T16:56:16.361Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/01e82a9dbd393631d4cdc403e6d2a13d0a5954cb6aa84fb7f837e5e0a6b958ce/operationStatuses/08584275811398670498\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/cx-backend-kv-access\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-3sr6ewk4usq2w\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-3sr6ewk4usq2w\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-kzzwmv6egrcdg\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-kzzwmv6egrcdg\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/cx-access-kzzwmv6egrcdg\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/cx-access-kzzwmv6egrcdg\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/cx-access-3sr6ewk4usq2w\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/cx-access-3sr6ewk4usq2w\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/cx-access-airyajwecijgk\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/cx-access-airyajwecijgk\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-airyajwecijgk\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-airyajwecijgk\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/msi-refresher-msi-kv-access\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-tpqc7lbi73vw6\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-tpqc7lbi73vw6\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-77z4miyuwcj5q\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-77z4miyuwcj5q\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-rskzz5hm4jrzk\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-rskzz5hm4jrzk\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035027859076026368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4305,"pr_state":"closed","pr_sha":"99364ef656afa8da443f2b41bcb4944465f70954","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-20T16:16:57Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3b348d5c7c83f07ea2fc1087617acffaa85e8a73ef89701e1120b4fbabb69e4a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035027859076026368","occurred_at":"2026-03-20T16:16:57Z","failed_at":"provision","signature_id":"c027c4d12f6a3d02006d0282f3d5b1020328ed77f7bf43d76c7a5e7566e656ac","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-03-20T16:43:07.752Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-03-20T16:43:10.492Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-03-20T16:43:12.010Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=15335b63c7222269b741262fad17b1eea805682b1397c6a9e0508ec3969cac63\ntime=2026-03-20T16:58:15.605Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j6026368-svc/providers/Microsoft.Resources/deployments/15335b63c7222269b741262fad17b1eea805682b1397c6a9e0508ec3969cac63/operationStatuses/08584275818942771261\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/aks-svc-kv-access-admin-api-cert-prow-j6026368\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/aks-svc-kv-access-frontend-cert-prow-j6026368\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/aksSPCRead-sessiongate-cert-prow-j6026368\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6026368-svc/providers/Microsoft.Resources/deployments/metrics-infra\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035025085672198144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4553,"pr_state":"closed","pr_sha":"7573bfbf6c5a2ea4b6ebbc79910eeabcb424ca64","final_merged_sha":"7573bfbf6c5a2ea4b6ebbc79910eeabcb424ca64","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T16:06:03Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4561/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035024502429061120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4561,"pr_state":"closed","pr_sha":"15ae132308931cc541a77791ad2f97ce6969b797","final_merged_sha":"15ae132308931cc541a77791ad2f97ce6969b797","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T16:03:43Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035019190368210944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4542,"pr_state":"closed","pr_sha":"52fc7d3a41cdc9d6774bfe7f5f45b5768abaf818","final_merged_sha":"52fc7d3a41cdc9d6774bfe7f5f45b5768abaf818","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T15:42:36Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4493/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035015450223448064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4493,"pr_state":"closed","pr_sha":"8a3e2673bc025c3e97823e6867d3b9ede213d29b","final_merged_sha":"8a3e2673bc025c3e97823e6867d3b9ede213d29b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T15:27:46Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4453/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035013567173890048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4453,"pr_state":"closed","pr_sha":"74066469c7ba25be3efcb17e4e6a47885ab78793","final_merged_sha":"74066469c7ba25be3efcb17e4e6a47885ab78793","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T15:20:17Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4560/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035008147067768832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4560,"pr_state":"closed","pr_sha":"2c8c865f07b50cfb22fe688f7b41c42368b93414","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-20T14:58:41Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"5da4ebcabe0f3e36f3bf98a4a6d185a6520acc80a07c8d38ebeef641d8b59dad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4560/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035008147067768832","occurred_at":"2026-03-20T14:58:41Z","failed_at":"e2e","signature_id":"3bef35098354a64bc1ea4c5cbf807e36e2c97d4b2e1724e3715261449eb9c1b2","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:120]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0014d29c0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mxzr74\" to finish creating: GET https://rp.j7768832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1cf7d87-0eb8-45b2-942c-277542c40fb0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1cf7d87-0eb8-45b2-942c-277542c40fb0\",\n \"name\": \"a1cf7d87-0eb8-45b2-942c-277542c40fb0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-20T15:47:06.242699653Z\",\n \"endTime\": \"2026-03-20T15:47:22.298584679Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"SAL validation failed with status code: 400\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-mxzr74\\\" to finish creating: GET https://rp.j7768832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1cf7d87-0eb8-45b2-942c-277542c40fb0\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1cf7d87-0eb8-45b2-942c-277542c40fb0\\\",\\n \\\"name\\\": \\\"a1cf7d87-0eb8-45b2-942c-277542c40fb0\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-20T15:47:06.242699653Z\\\",\\n \\\"endTime\\\": \\\"2026-03-20T15:47:22.298584679Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"SAL validation failed with status code: 400\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0014d29a0\u003e{\n msg: \"failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-mxzr74\\\" to finish creating: GET https://rp.j7768832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1cf7d87-0eb8-45b2-942c-277542c40fb0\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1cf7d87-0eb8-45b2-942c-277542c40fb0\\\",\\n \\\"name\\\": \\\"a1cf7d87-0eb8-45b2-942c-277542c40fb0\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-20T15:47:06.242699653Z\\\",\\n \\\"endTime\\\": \\\"2026-03-20T15:47:22.298584679Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"SAL validation failed with status code: 400\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0014f8c60\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"2de212f1-7a31-4e59-8625-db2c8547d4f7\",\n ],\n \"Date\": [\n \"Fri, 20 Mar 2026 15:47:26 GMT\",\n ],\n \"Content-Length\": [\"481\"],\n \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0014d28a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:120]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0014d29c0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mxzr74\" to finish creating: GET https://rp.j7768832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1cf7d87-0eb8-45b2-942c-277542c40fb0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1cf7d87-0eb8-45b2-942c-277542c40fb0\",\n \"name\": \"a1cf7d87-0eb8-45b2-942c-277542c40fb0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-20T15:47:06.242699653Z\",\n \"endTime\": \"2026-03-20T15:47:22.298584679Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"SAL validation failed with status code: 400\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-mxzr74\\\" to finish creating: GET https://rp.j7768832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1cf7d87-0eb8-45b2-942c-277542c40fb0\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1cf7d87-0eb8-45b2-942c-277542c40fb0\\\",\\n \\\"name\\\": \\\"a1cf7d87-0eb8-45b2-942c-277542c40fb0\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-20T15:47:06.242699653Z\\\",\\n \\\"endTime\\\": \\\"2026-03-20T15:47:22.298584679Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"SAL validation failed with status code: 400\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0014d29a0\u003e{\n msg: \"failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-mxzr74\\\" to finish creating: GET https://rp.j7768832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1cf7d87-0eb8-45b2-942c-277542c40fb0\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1cf7d87-0eb8-45b2-942c-277542c40fb0\\\",\\n \\\"name\\\": \\\"a1cf7d87-0eb8-45b2-942c-277542c40fb0\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-20T15:47:06.242699653Z\\\",\\n \\\"endTime\\\": \\\"2026-03-20T15:47:22.298584679Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"SAL validation failed with status code: 400\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0014f8c60\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"2de212f1-7a31-4e59-8625-db2c8547d4f7\",\n ],\n \"Date\": [\n \"Fri, 20 Mar 2026 15:47:26 GMT\",\n ],\n \"Content-Length\": [\"481\"],\n \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0014d28a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"97bc7a9140cde99d4476c4f7f132eae7b10269d13c7b09b073c80a436425231b","failure_pattern":"ERROR CODE: InternalServerError; detail message SAL validation failed with status code: 400; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cccb7a8e127a6e9aa9c209ea08e52988410485aac95fe1d137600c2b692121ba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4560/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035008147067768832","occurred_at":"2026-03-20T14:58:41Z","failed_at":"e2e","signature_id":"3369796889b7cb59b08cd09c2bbfd5e5f449b04e2293f8f293e15dc622fac674","test_name":"Customer should be able to create an HCP cluster with private key vault","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/keyvault_private.go:105]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b08f40\u003e: \n failed to create HCP cluster private-key-vault: failed waiting for cluster=\"private-key-vault\" in resourcegroup=\"private-key-vault-mbcpfw\" to finish creating: GET https://rp.j7768832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03fab8fe-4421-4f9f-886d-eea23eadbe94\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03fab8fe-4421-4f9f-886d-eea23eadbe94\",\n \"name\": \"03fab8fe-4421-4f9f-886d-eea23eadbe94\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-20T15:46:52.395992159Z\",\n \"endTime\": \"2026-03-20T15:47:12.443442695Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"SAL validation failed with status code: 400\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster private-key-vault: failed waiting for cluster=\\\"private-key-vault\\\" in resourcegroup=\\\"private-key-vault-mbcpfw\\\" to finish creating: GET https://rp.j7768832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03fab8fe-4421-4f9f-886d-eea23eadbe94\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03fab8fe-4421-4f9f-886d-eea23eadbe94\\\",\\n \\\"name\\\": \\\"03fab8fe-4421-4f9f-886d-eea23eadbe94\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-20T15:46:52.395992159Z\\\",\\n \\\"endTime\\\": \\\"2026-03-20T15:47:12.443442695Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"SAL validation failed with status code: 400\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000b08f20\u003e{\n msg: \"failed waiting for cluster=\\\"private-key-vault\\\" in resourcegroup=\\\"private-key-vault-mbcpfw\\\" to finish creating: GET https://rp.j7768832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03fab8fe-4421-4f9f-886d-eea23eadbe94\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03fab8fe-4421-4f9f-886d-eea23eadbe94\\\",\\n \\\"name\\\": \\\"03fab8fe-4421-4f9f-886d-eea23eadbe94\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-20T15:46:52.395992159Z\\\",\\n \\\"endTime\\\": \\\"2026-03-20T15:47:12.443442695Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"SAL validation failed with status code: 400\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00121b620\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"efbd221b-3753-44b9-920e-1119860e1071\",\n ],\n \"Date\": [\n \"Fri, 20 Mar 2026 15:47:12 GMT\",\n ],\n \"Content-Length\": [\"481\"],\n \"X-Envoy-Upstream-Service-Time\": [\"9\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000b08d00\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 11...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/keyvault_private.go:105]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b08f40\u003e: \n failed to create HCP cluster private-key-vault: failed waiting for cluster=\"private-key-vault\" in resourcegroup=\"private-key-vault-mbcpfw\" to finish creating: GET https://rp.j7768832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03fab8fe-4421-4f9f-886d-eea23eadbe94\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03fab8fe-4421-4f9f-886d-eea23eadbe94\",\n \"name\": \"03fab8fe-4421-4f9f-886d-eea23eadbe94\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-20T15:46:52.395992159Z\",\n \"endTime\": \"2026-03-20T15:47:12.443442695Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"SAL validation failed with status code: 400\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster private-key-vault: failed waiting for cluster=\\\"private-key-vault\\\" in resourcegroup=\\\"private-key-vault-mbcpfw\\\" to finish creating: GET https://rp.j7768832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03fab8fe-4421-4f9f-886d-eea23eadbe94\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03fab8fe-4421-4f9f-886d-eea23eadbe94\\\",\\n \\\"name\\\": \\\"03fab8fe-4421-4f9f-886d-eea23eadbe94\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-20T15:46:52.395992159Z\\\",\\n \\\"endTime\\\": \\\"2026-03-20T15:47:12.443442695Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"SAL validation failed with status code: 400\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000b08f20\u003e{\n msg: \"failed waiting for cluster=\\\"private-key-vault\\\" in resourcegroup=\\\"private-key-vault-mbcpfw\\\" to finish creating: GET https://rp.j7768832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03fab8fe-4421-4f9f-886d-eea23eadbe94\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03fab8fe-4421-4f9f-886d-eea23eadbe94\\\",\\n \\\"name\\\": \\\"03fab8fe-4421-4f9f-886d-eea23eadbe94\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-20T15:46:52.395992159Z\\\",\\n \\\"endTime\\\": \\\"2026-03-20T15:47:12.443442695Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"SAL validation failed with status code: 400\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00121b620\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"efbd221b-3753-44b9-920e-1119860e1071\",\n ],\n \"Date\": [\n \"Fri, 20 Mar 2026 15:47:12 GMT\",\n ],\n \"Content-Length\": [\"481\"],\n \"X-Envoy-Upstream-Service-Time\": [\"9\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000b08d00\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 11...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"97bc7a9140cde99d4476c4f7f132eae7b10269d13c7b09b073c80a436425231b","failure_pattern":"ERROR CODE: InternalServerError; detail message SAL validation failed with status code: 400; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["97bc7a9140cde99d4476c4f7f132eae7b10269d13c7b09b073c80a436425231b"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4559/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035007398896209920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4559,"pr_state":"closed","pr_sha":"5451b94b33d9f18f584a5120a1f83f2efdf0a65e","final_merged_sha":"88537065778ffab09a3fb724a338fca3bda8a0d9","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-20T14:55:44Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035005363601805312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4503,"pr_state":"closed","pr_sha":"dec83f9fbfccef278c235e39666e1b3c5d564a26","final_merged_sha":"3a9b5c9cb4bcc81a28d2a98b1dd50a3c686f225a","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-20T14:47:41Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4493/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035000016732426240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4493,"pr_state":"closed","pr_sha":"8a3e2673bc025c3e97823e6867d3b9ede213d29b","final_merged_sha":"8a3e2673bc025c3e97823e6867d3b9ede213d29b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-20T14:26:19Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"7e0cd59f5941c701c5d1891975e46dfbec9225e5725edde099433b4c003a8e47","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4493/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2035000016732426240","occurred_at":"2026-03-20T14:26:19Z","failed_at":"provision","signature_id":"963b6b4b5188db91f3076c5e980a1758634b02c277e107ef32ef8d20e303b0b7","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.AdminAPI/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-03-20T15:10:05.652Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=global step=mirror-image description=\"Step mirror-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e adminApi.image.registry \u003cnil\u003e}:{\u003cnil\u003e adminApi.image.repository \u003cnil\u003e}@{\u003cnil\u003e adminApi.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-20T15:10:47.848Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org\\nSetting up registry authentication for CI source registry.\\ninfo: Using registry public hostname registry.build02.ci.openshift.org\\nSaved credentials for registry.build02.ci.openshift.org into /tmp/tmp.kN6XNGprsf/containers/auth.json\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.build02.ci.openshift.org/ci-op-c5iz4fkw/pipeline@sha256:60efa6e90e760e999ed57c877081e9e1149ac3cd70268aa50f5e673edcec77dd to arohcpsvcdev.azurecr.io/ci-op-c5iz4fkw/pipeline:60efa6e90e760e999ed57c877081e9e1149ac3cd70268aa50f5e673edcec77dd.\\nThe image will still be available under it's original digest sha256:60efa6e90e760e999ed57c877081e9e1149ac3cd70268aa50f5e673edcec77dd in the target registry.\\nError: Put \\\"https://arohcpsvcdev.azurecr.io/v2/ci-op-c5iz4fkw/pipeline/blobs/uploads/15743118-8363-4850-8421-567ab29be8bf?_nouploadcache=false\u0026_state=yL6O1Mkyse92xO6o0N7_ktVbhzUvvfcwZXvcxCNKiY17Ik5hbWUiOiJjaS1vcC1jNWl6NGZrdy9waXBlbGluZSIsIlVVSUQiOiIxNTc0MzExOC04MzYzLTQ4NTAtODQyMS01NjdhYjI5YmU4YmYiLCJPZmZzZXQiOjAsIlN0YXJ0ZWRBdCI6IjIwMjYtMDMtMjBUMTU6MTA6MTcuMzU3NTk1Mjk5WiJ9\u0026digest=sha256%3A78a154f385e7d87fb9adfc808c2f030191c9b14cb2f9df801a6546418c3a47f9\\\": write tcp 10.130.136.56:50570-\u003e20.62.128.0:443: use of closed network connection\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"feb51bfad74887d1c8b65a30577cd804042a298c27775cf1dd4bb9265c464069","failure_pattern":"Error: Put \"\u003curl\u003e\": write tcp \u003cip\u003e-\u003e\u003cip\u003e: use of closed network connection","search_query":"Error: Put \\\"https://arohcpsvcdev.azurecr.io/v2/ci-op-c5iz4fkw/pipeline/blobs/uploads/15743118-8363-4850-8421-567ab29be8bf?_nouploadcache=false\u0026_state=yL6O1Mkyse92xO6o0N7_ktVbhzUvvfcwZXvcxCNKiY17Ik5hbWUiOiJjaS1vcC1jNWl6NGZrdy9waXBlbGluZSIsIlVVSUQiOiIxNTc0MzExOC04MzYzLTQ4NTAtODQyMS01NjdhYjI5YmU4YmYiLCJPZmZzZXQiOjAsIlN0YXJ0ZWRBdCI6IjIwMjYtMDMtMjBUMTU6MTA6MTcuMzU3NTk1Mjk5WiJ9\u0026digest=sha256%3A78a154f385e7d87fb9adfc808c2f030191c9b14cb2f9df801a6546418c3a47f9\\\": write tcp 10.130.136.56:50570-\u003e20.62.128.0:443: use of closed network connection"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["feb51bfad74887d1c8b65a30577cd804042a298c27775cf1dd4bb9265c464069"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034999313234399232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4493,"pr_state":"closed","pr_sha":"8a3e2673bc025c3e97823e6867d3b9ede213d29b","final_merged_sha":"8a3e2673bc025c3e97823e6867d3b9ede213d29b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-20T14:23:39Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"5bbe2cf9696730b7e4ec2ce34d7af73b0de48ba33f666fd639580e3b6cad11fc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034999313234399232","occurred_at":"2026-03-20T14:23:39Z","failed_at":"provision","signature_id":"cd8084006defc9c13ad89c33a95b9e9d50712ac6dca0c813c1055275a5ba4139","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Frontend/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-03-20T15:04:46.129Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=global step=mirror-image description=\"Step mirror-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e frontend.image.registry \u003cnil\u003e}:{\u003cnil\u003e frontend.image.repository \u003cnil\u003e}@{\u003cnil\u003e frontend.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-20T15:07:28.935Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org\\nSetting up registry authentication for CI source registry.\\ninfo: Using registry public hostname registry.build02.ci.openshift.org\\nSaved credentials for registry.build02.ci.openshift.org into /tmp/tmp.tGfzCZRWRe/containers/auth.json\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.build02.ci.openshift.org/ci-op-ddghffln/pipeline@sha256:5e8699aeeb3a8491206ef4a364616b18c4c964d854a38ebf8f1ac985dbffbb13 to arohcpsvcdev.azurecr.io/ci-op-ddghffln/pipeline:5e8699aeeb3a8491206ef4a364616b18c4c964d854a38ebf8f1ac985dbffbb13.\\nThe image will still be available under it's original digest sha256:5e8699aeeb3a8491206ef4a364616b18c4c964d854a38ebf8f1ac985dbffbb13 in the target registry.\\nError: Post \\\"https://arohcpsvcdev.azurecr.io/v2/ci-op-ddghffln/pipeline/blobs/uploads/\\\": EOF\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"84254946c8bbab451d344c5f10355c3824c6ad1da91040ccc3a80ff8b5ac1817","failure_pattern":"Error: Post \"\u003curl\u003e\": EOF","search_query":"Error: Post \\\"https://arohcpsvcdev.azurecr.io/v2/ci-op-ddghffln/pipeline/blobs/uploads/\\\": EOF"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["84254946c8bbab451d344c5f10355c3824c6ad1da91040ccc3a80ff8b5ac1817"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034995093194149888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4493,"pr_state":"closed","pr_sha":"8a3e2673bc025c3e97823e6867d3b9ede213d29b","final_merged_sha":"8a3e2673bc025c3e97823e6867d3b9ede213d29b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T14:06:53Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034994799680950272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4477,"pr_state":"closed","pr_sha":"165c4971b2b2c62ad38cac720899cb6c393d1e21","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-20T14:05:42Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"15e77f90ccfdbee562f96afe5a9dbc551c1f744d2471f787ab3629bee77c82ac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034994799680950272","occurred_at":"2026-03-20T14:05:42Z","failed_at":"provision","signature_id":"a325c582a7824d3df410264c5f84a052c3daab42753520b21a7eef360027948f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-03-20T14:49:43.292Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=global step=mirror-image description=\"Step mirror-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e backend.image.registry \u003cnil\u003e}:{\u003cnil\u003e backend.image.repository \u003cnil\u003e}@{\u003cnil\u003e backend.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-20T14:49:55.346Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org\\nSetting up registry authentication for CI source registry.\\ninfo: Using registry public hostname registry.build02.ci.openshift.org\\nSaved credentials for registry.build02.ci.openshift.org into /tmp/tmp.6nYQLImg11/containers/auth.json\\nLogging into target ACR arohcpsvcdev.\\nFailed to log in to ACR arohcpsvcdev: ERROR: Could not connect to the registry login server 'arohcpsvcdev.azurecr.io'. Please verify that the registry exists and the URL 'https://arohcpsvcdev.azurecr.io/v2/' is reachable from your environment.\\nTry running 'az acr check-health -n arohcpsvcdev --yes' to diagnose this issue.\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"061908f35a0fa01699ae0bafb53b5a8e0ffd0428c6990d52950e0f0f82a49466","failure_pattern":"Failed to log in to ACR arohcpsvcdev: ERROR: Could not connect to the registry login server 'arohcpsvcdev.azurecr.io'. Please verify that the registry exists and the URL '\u003curl\u003e' is reachable from your environment.","search_query":"Failed to log in to ACR arohcpsvcdev: ERROR: Could not connect to the registry login server 'arohcpsvcdev.azurecr.io'. Please verify that the registry exists and the URL 'https://arohcpsvcdev.azurecr.io/v2/' is reachable from your environment."}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["061908f35a0fa01699ae0bafb53b5a8e0ffd0428c6990d52950e0f0f82a49466"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034988785971761152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4305,"pr_state":"closed","pr_sha":"99364ef656afa8da443f2b41bcb4944465f70954","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-20T13:41:43Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"ced6e76928638f048ef92b895315cede1980ea39bbc2666a567af7d271933158","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034988785971761152","occurred_at":"2026-03-20T13:41:43Z","failed_at":"e2e","signature_id":"06c1b87a87f5fdba0af991106262ecf4d387d0766a64fe7ed1163d987b6e72d8","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:281]: Timed out after 30.001s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000b6e440\u003e: \n read access verification failed: get pods -n kube-system: pods is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"pods\" in API group \"\" in the namespace \"kube-system\"\n get configmaps -n kube-system: configmaps is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"configmaps\" in API group \"\" in the namespace \"kube-system\"\n {\n msg: \"read access verification failed: get pods -n kube-system: pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\\nget configmaps -n kube-system: configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.joinError | 0xc0002110f8\u003e{\n errs: [\n \u003c*fmt.wrapError | 0xc000b6e160\u003e{\n msg: \"get pods -n kube-system: pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.StatusError | 0xc000b1d2c0\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n Reason: \"Forbidden\",\n Details: {Name: \"\", Group: \"\", Kind: \"pods\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n },\n },\n \u003c*fmt.wrapError | 0xc000b6e3e0\u003e{\n msg: \"get configmaps -n kube-system: configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.StatusError | 0xc000b1d540\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n Reason: \"Forbidden\",\n Details: {Name: \"\", Group: \"\", Kind: \"configmaps\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n },\n },\n ],\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:281]: Timed out after 30.001s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000b6e440\u003e: \n read access verification failed: get pods -n kube-system: pods is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"pods\" in API group \"\" in the namespace \"kube-system\"\n get configmaps -n kube-system: configmaps is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"configmaps\" in API group \"\" in the namespace \"kube-system\"\n {\n msg: \"read access verification failed: get pods -n kube-system: pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\\nget configmaps -n kube-system: configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.joinError | 0xc0002110f8\u003e{\n errs: [\n \u003c*fmt.wrapError | 0xc000b6e160\u003e{\n msg: \"get pods -n kube-system: pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.StatusError | 0xc000b1d2c0\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n Reason: \"Forbidden\",\n Details: {Name: \"\", Group: \"\", Kind: \"pods\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n },\n },\n \u003c*fmt.wrapError | 0xc000b6e3e0\u003e{\n msg: \"get configmaps -n kube-system: configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.StatusError | 0xc000b1d540\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n Reason: \"Forbidden\",\n Details: {Name: \"\", Group: \"\", Kind: \"configmaps\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n },\n },\n ],\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3c6a69913a4e7e3ddde30c4ad0c3578c4319954e53163360fc7432ced31dda4a","failure_pattern":"read access verification failed: get pod \u003cpod\u003e kube-system: pods is forbidden: User \"system:sre-break-glass:\u003cuuid\u003e\" cannot list resource \"pods\" in API group \"\" in the namespace \"\u003cnamespace\u003e\"","search_query":"read access verification failed: get pods -n kube-system: pods is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"pods\" in API group \"\" in the namespace \"kube-system\""}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["3c6a69913a4e7e3ddde30c4ad0c3578c4319954e53163360fc7432ced31dda4a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4558/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034987999338434560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4558,"pr_state":"closed","pr_sha":"1205c8fffd91fbcc73a425533f183d045beea0a7","final_merged_sha":"1205c8fffd91fbcc73a425533f183d045beea0a7","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T13:38:37Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034986795292168192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4557,"pr_state":"closed","pr_sha":"d73d64e4a7c81445054dba60cac569e834d92458","final_merged_sha":"d73d64e4a7c81445054dba60cac569e834d92458","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T13:33:46Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4370/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034984093887762432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4370,"pr_state":"open","pr_sha":"1bbbcbfa4f48d1dbfa9897cfe896ade690b77d5c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-20T13:23:10Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034983415274541056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4305,"pr_state":"closed","pr_sha":"29205a2f1eb827313fc339d625c23f2edacdef97","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-20T13:20:24Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"a2ddec913c95fcb92f452e23e538edcca732a824889426bcae82b648ec73ae64","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034983415274541056","occurred_at":"2026-03-20T13:20:24Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4494/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034982588837269504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4494,"pr_state":"closed","pr_sha":"8b266f328ee22ecc9662ce597b0de6d1714bc9e8","final_merged_sha":"b6d9c642ca814d40f9bb7032fbbf894bc4edfd69","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-20T13:17:11Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034982289141665792","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4493,"pr_state":"closed","pr_sha":"8a3e2673bc025c3e97823e6867d3b9ede213d29b","final_merged_sha":"8a3e2673bc025c3e97823e6867d3b9ede213d29b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-20T13:15:49Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"27c8a4ded5fc2c275ac043d13cf3b08020a1dca53a637418d80b80589567ba15","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034982289141665792","occurred_at":"2026-03-20T13:15:49Z","failed_at":"provision","signature_id":"9df96cedd304ce960ec46d168dc0a0b5c62065ee44735bac356e14225c792ada","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Agent/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-03-20T13:55:23.479Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=global step=mirror-image description=\"Step mirror-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e maestro.image.registry \u003cnil\u003e}:{\u003cnil\u003e maestro.image.repository \u003cnil\u003e}@{\u003cnil\u003e maestro.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-20T13:55:47.157Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org\\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nFailed to log in to ACR arohcpsvcdev: ERROR: The command failed with an unexpected error. Here is the traceback:\\nERROR: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 204, in _new_conn\\n sock = connection.create_connection(\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/util/connection.py\\\", line 60, in create_connection\\n for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/python3.12/socket.py\\\", line 963, in getaddrinfo\\n for res in _socket.getaddrinfo(host, port, family, type, proto, flags):\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\nsocket.gaierror: [Errno -2] Name or service not known\\n\\nThe above exception was the direct cause of the following exception:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 787, in urlopen\\n response = self._make_request(\\n ^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 488, in _make_request\\n raise new_e\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 464, in _make_request\\n self._validate_conn(conn)\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 1093, in _validate_conn\\n conn.connect()\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 759, in connect\\n self.sock = sock = self._new_conn()\\n ^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 211, in _new_conn\\n raise NameResolutionError(self.host, self, e) from e\\nurllib3.exceptions.NameResolutionError: HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\n\\nThe above exception was the direct cause of the following exception:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/adapters.py\\\", line 667, in send\\n resp = conn.urlopen(\\n ^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 841, in urlopen\\n retries = retries.increment(\\n ^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/util/retry.py\\\", line 535, in increment\\n raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\nurllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\n\\nDuring handling of the above exception, another exception occurred:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/knack/cli.py\\\", line 233, in invoke\\n cmd_result = self.invocation.execute(args)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 682, in execute\\n raise ex\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 812, in _run_jobs_serially\\n results.append(self._run_job(expanded_arg, cmd_copy))\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 781, in _run_job\\n result = cmd_copy(params)\\n ^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 336, in __call__\\n return self.handler(*args, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/command_operation.py\\\", line 120, in handler\\n return op(**command_args)\\n ^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/custom.py\\\", line 294, in acr_login\\n login_server, _, password = get_login_credentials(\\n ^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 505, in get_login_credentials\\n return _get_credentials(cmd,\\n ^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 449, in _get_credentials\\n return login_server, EMPTY_GUID, _get_aad_token(cli_ctx,\\n ^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 276, in _get_aad_token\\n token_params = _handle_challenge_phase(\\n ^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 100, in _handle_challenge_phase\\n challenge = requests.get(request_url, verify=not should_disable_connection_verify())\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/api.py\\\", line 73, in get\\n return request(\\\"get\\\", url, params=params, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/api.py\\\", line 59, in request\\n return session.request(method=method, url=url, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/sessions.py\\\", line 589, in request\\n resp = self.send(prep, **send_kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/sessions.py\\\", line 703, in send\\n r = adapter.send(request, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/adapters.py\\\", line 700, in send\\n raise ConnectionError(e, request=request)\\nrequests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\nTo check existing issues, please visit: https://github.com/Azure/azure-cli/issues\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ee5b1a1a3315ac34a62d3e4a95293e1f5e2f2b1ef572b3ff8359f4b3944b63b3","failure_pattern":"requests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve","search_query":"requests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ee5b1a1a3315ac34a62d3e4a95293e1f5e2f2b1ef572b3ff8359f4b3944b63b3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034979682243317760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4305,"pr_state":"closed","pr_sha":"6045537711c0284463ec9131d962ab4f96b9974e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-20T13:05:35Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"a26442919b3a3e148ad91093d4f9e9d291989a0ed4937dd3c5a2d9b56e0d9760","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034979682243317760","occurred_at":"2026-03-20T13:05:35Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3907/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034978465924517888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3907,"pr_state":"closed","pr_sha":"bff24f16285eb14657ded302ac0a597b45cbfbfb","final_merged_sha":"e9ca15da6d935a378fb165189c2441a771314f73","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-20T13:00:48Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4542/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034973040885895168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4542,"pr_state":"closed","pr_sha":"52fc7d3a41cdc9d6774bfe7f5f45b5768abaf818","final_merged_sha":"52fc7d3a41cdc9d6774bfe7f5f45b5768abaf818","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T12:39:15Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4552/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034970479634157568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4552,"pr_state":"closed","pr_sha":"4d0bbb46e92cc39969c8b1c8ed91f0dd35f385d7","final_merged_sha":"4d0bbb46e92cc39969c8b1c8ed91f0dd35f385d7","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T12:29:04Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4513/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034959658539552768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4513,"pr_state":"open","pr_sha":"77f2b6e6a2041e8f95a5d862d8cf78122a0dce5b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-20T11:46:04Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4555/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034950792561037312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4555,"pr_state":"closed","pr_sha":"a2764379b7de5120ecc35aa326b6b1c21a551168","final_merged_sha":"a2764379b7de5120ecc35aa326b6b1c21a551168","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T11:10:49Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034943357746155520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3661,"pr_state":"closed","pr_sha":"6029601d96d68c2ea706cbf5922d5348b46bc432","final_merged_sha":"6029601d96d68c2ea706cbf5922d5348b46bc432","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T10:41:18Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034942692164636672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3661,"pr_state":"closed","pr_sha":"6029601d96d68c2ea706cbf5922d5348b46bc432","final_merged_sha":"6029601d96d68c2ea706cbf5922d5348b46bc432","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T10:38:36Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4542/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034934756579217408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4542,"pr_state":"closed","pr_sha":"ebf14419a67e0180a48660ab0f905b125e23c157","final_merged_sha":"52fc7d3a41cdc9d6774bfe7f5f45b5768abaf818","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-20T10:07:07Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3907/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034930846296707072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3907,"pr_state":"closed","pr_sha":"dc915704fad4828c797314528f99785046114d7a","final_merged_sha":"e9ca15da6d935a378fb165189c2441a771314f73","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-20T09:51:35Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4494/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034928579812265984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4494,"pr_state":"closed","pr_sha":"826640a9e121ffd75ad2b5a576023e68c62a9565","final_merged_sha":"b6d9c642ca814d40f9bb7032fbbf894bc4edfd69","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-20T09:42:35Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"9e8c1f092963a512c7d8b1de82d329c0147700c8bdc7e6220fd284aec67a88b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4494/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034928579812265984","occurred_at":"2026-03-20T09:42:35Z","failed_at":"e2e","signature_id":"f85dfd8836dbb5a604f6357f7fec3a250270fa2ff2c16796821677c317f4f595","test_name":"Engineering should be able to retrieve expected metrics from the /metrics endpoint","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:74]: Failed to set up port-forward\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004bd080\u003e: \n port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.10:10250, code 500: 500 Internal Server Error\n {\n msg: \"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.10:10250, code 500: 500 Internal Server Error\",\n err: \u003c*errors.errorString | 0xc0008ef350\u003e{\n s: \"error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.10:10250, code 500: 500 Internal Server Error\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:74]: Failed to set up port-forward\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004bd080\u003e: \n port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.10:10250, code 500: 500 Internal Server Error\n {\n msg: \"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.10:10250, code 500: 500 Internal Server Error\",\n err: \u003c*errors.errorString | 0xc0008ef350\u003e{\n s: \"error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.10:10250, code 500: 500 Internal Server Error\",\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f27efe24b36721c3037189b0270b3c037f3dcd8e282db885731d7395d97e1e2a","failure_pattern":"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing \u003cip\u003e:\u003cport\u003e, code 500: 500 Internal Server Error","search_query":"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.10:10250, code 500: 500 Internal Server Error"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f27efe24b36721c3037189b0270b3c037f3dcd8e282db885731d7395d97e1e2a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034927893695434752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4554,"pr_state":"closed","pr_sha":"861babe0af4cd1f8dc00e5055c092379ae4c0362","final_merged_sha":"cab9df44f4aaadfa6301e36bed1fb8403e84a43b","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-20T09:39:51Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034914492575387648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4347,"pr_state":"closed","pr_sha":"a82c62541bf569a06efb0a6211c6ae9e8a2c01a4","final_merged_sha":"a82c62541bf569a06efb0a6211c6ae9e8a2c01a4","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T08:46:36Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4541/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034903808198316032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4541,"pr_state":"closed","pr_sha":"364fb5c63fca1e0cbf62bd6496b150c1c7d401dd","final_merged_sha":"364fb5c63fca1e0cbf62bd6496b150c1c7d401dd","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-20T08:04:09Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"5087c786fa4ca5cc19033dbb91777cc07d088dd6c4148a3f1f3f9aaf6eae38ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4541/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034903808198316032","occurred_at":"2026-03-20T08:04:09Z","failed_at":"e2e","signature_id":"19e5d6fe65fcb71da042a306ea2959c6e592a672187d9351596b71164b56b7aa","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:245]: Unexpected error:\n \u003c*errors.joinError | 0xc00102cae0\u003e: \n found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-8x9hrr\n {\n errs: [\n \u003c*errors.errorString | 0xc0015bede0\u003e{\n s: \"found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-8x9hrr\",\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:245]: Unexpected error:\n \u003c*errors.joinError | 0xc00102cae0\u003e: \n found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-8x9hrr\n {\n errs: [\n \u003c*errors.errorString | 0xc0015bede0\u003e{\n s: \"found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-8x9hrr\",\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c922edc02b3f5a976a854a17e119a5c6a54606ad1581851b8fd7764114448aa4","failure_pattern":"found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-\u003cid\u003e","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c922edc02b3f5a976a854a17e119a5c6a54606ad1581851b8fd7764114448aa4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4385/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034900732808794112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4385,"pr_state":"closed","pr_sha":"07e421bae19659fd95e6d16899bed8ad2fda3863","final_merged_sha":"07e421bae19659fd95e6d16899bed8ad2fda3863","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T07:51:55Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4532/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034886993057419264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4532,"pr_state":"closed","pr_sha":"8033aacf4605ee2e3dd591bd5116c0e8c0a2b5f2","final_merged_sha":"8033aacf4605ee2e3dd591bd5116c0e8c0a2b5f2","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T06:57:18Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034886916456845312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4347,"pr_state":"closed","pr_sha":"a82c62541bf569a06efb0a6211c6ae9e8a2c01a4","final_merged_sha":"a82c62541bf569a06efb0a6211c6ae9e8a2c01a4","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T06:57:01Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034886614970273792","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4503,"pr_state":"closed","pr_sha":"16b44df65e798f2ce166f58b4b3cdc45fae3ccec","final_merged_sha":"3a9b5c9cb4bcc81a28d2a98b1dd50a3c686f225a","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-20T06:55:49Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034886419553456128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3661,"pr_state":"closed","pr_sha":"6029601d96d68c2ea706cbf5922d5348b46bc432","final_merged_sha":"6029601d96d68c2ea706cbf5922d5348b46bc432","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T06:55:03Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034882825383055360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4553,"pr_state":"closed","pr_sha":"13d99329efd6f8793c66c3a62fa9e8a29e36fc03","final_merged_sha":"7573bfbf6c5a2ea4b6ebbc79910eeabcb424ca64","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-20T06:40:46Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"40b5bd1607531b433e05739bfe7e4b72a78e960bfc84ff9063453130328eee45","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034882825383055360","occurred_at":"2026-03-20T06:40:46Z","failed_at":"provision","signature_id":"75a1070e38a784210f91dbc5392b58a7bc4adfcd8c9955839cc6a48f6241ff24","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-03-20T07:16:55.672Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image description=\"Step mirror-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e secretSyncController.image.registry \u003cnil\u003e}:{\u003cnil\u003e secretSyncController.image.repository \u003cnil\u003e}@{\u003cnil\u003e secretSyncController.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-20T07:17:18.851Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org\\nFetch pull secret for source registry registry.k8s.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nFailed to log in to ACR arohcpsvcdev: ERROR: Could not connect to the registry login server 'arohcpsvcdev.azurecr.io'. Please verify that the registry exists and the URL 'https://arohcpsvcdev.azurecr.io/v2/' is reachable from your environment.\\nTry running 'az acr check-health -n arohcpsvcdev --yes' to diagnose this issue.\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"061908f35a0fa01699ae0bafb53b5a8e0ffd0428c6990d52950e0f0f82a49466","failure_pattern":"Failed to log in to ACR arohcpsvcdev: ERROR: Could not connect to the registry login server 'arohcpsvcdev.azurecr.io'. Please verify that the registry exists and the URL '\u003curl\u003e' is reachable from your environment.","search_query":"Failed to log in to ACR arohcpsvcdev: ERROR: Could not connect to the registry login server 'arohcpsvcdev.azurecr.io'. Please verify that the registry exists and the URL 'https://arohcpsvcdev.azurecr.io/v2/' is reachable from your environment."}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["061908f35a0fa01699ae0bafb53b5a8e0ffd0428c6990d52950e0f0f82a49466"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4552/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034880093335064576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4552,"pr_state":"closed","pr_sha":"8d509c6f635422f40da9cb6f23bbc31e27d155c9","final_merged_sha":"4d0bbb46e92cc39969c8b1c8ed91f0dd35f385d7","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-20T06:29:55Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"7fe02546f095d9fdd2c45f1a1c5b715d8c8301e90f3327c7377bd8a9f1615e97","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4552/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034880093335064576","occurred_at":"2026-03-20T06:29:55Z","failed_at":"provision","signature_id":"416b0b3d1f2dcc7ada871ed732bdb46f6c998121884f47406ec4f5cd31ef36a2","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RouteMonitorOperator/global/mirror-operator-image","test_suite":"step graph","failure_text":"time=2026-03-20T07:08:18.552Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=global step=mirror-operator-image description=\"Step mirror-operator-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e routeMonitorOperator.operatorImage.registry \u003cnil\u003e}:{\u003cnil\u003e routeMonitorOperator.operatorImage.repository \u003cnil\u003e}@{\u003cnil\u003e routeMonitorOperator.operatorImage.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-20T07:08:32.083Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=global step=mirror-operator-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org\\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nFailed to log in to ACR arohcpsvcdev: ERROR: The command failed with an unexpected error. Here is the traceback:\\nERROR: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 204, in _new_conn\\n sock = connection.create_connection(\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/util/connection.py\\\", line 60, in create_connection\\n for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/python3.12/socket.py\\\", line 963, in getaddrinfo\\n for res in _socket.getaddrinfo(host, port, family, type, proto, flags):\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\nsocket.gaierror: [Errno -2] Name or service not known\\n\\nThe above exception was the direct cause of the following exception:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 787, in urlopen\\n response = self._make_request(\\n ^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 488, in _make_request\\n raise new_e\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 464, in _make_request\\n self._validate_conn(conn)\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 1093, in _validate_conn\\n conn.connect()\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 759, in connect\\n self.sock = sock = self._new_conn()\\n ^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 211, in _new_conn\\n raise NameResolutionError(self.host, self, e) from e\\nurllib3.exceptions.NameResolutionError: HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\n\\nThe above exception was the direct cause of the following exception:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/adapters.py\\\", line 667, in send\\n resp = conn.urlopen(\\n ^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 841, in urlopen\\n retries = retries.increment(\\n ^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/util/retry.py\\\", line 535, in increment\\n raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\nurllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\n\\nDuring handling of the above exception, another exception occurred:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/knack/cli.py\\\", line 233, in invoke\\n cmd_result = self.invocation.execute(args)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 682, in execute\\n raise ex\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 812, in _run_jobs_serially\\n results.append(self._run_job(expanded_arg, cmd_copy))\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 781, in _run_job\\n result = cmd_copy(params)\\n ^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 336, in __call__\\n return self.handler(*args, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/command_operation.py\\\", line 120, in handler\\n return op(**command_args)\\n ^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/custom.py\\\", line 294, in acr_login\\n login_server, _, password = get_login_credentials(\\n ^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 505, in get_login_credentials\\n return _get_credentials(cmd,\\n ^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 449, in _get_credentials\\n return login_server, EMPTY_GUID, _get_aad_token(cli_ctx,\\n ^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 276, in _get_aad_token\\n token_params = _handle_challenge_phase(\\n ^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 100, in _handle_challenge_phase\\n challenge = requests.get(request_url, verify=not should_disable_connection_verify())\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/api.py\\\", line 73, in get\\n return request(\\\"get\\\", url, params=params, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/api.py\\\", line 59, in request\\n return session.request(method=method, url=url, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/sessions.py\\\", line 589, in request\\n resp = self.send(prep, **send_kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/sessions.py\\\", line 703, in send\\n r = adapter.send(request, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/adapters.py\\\", line 700, in send\\n raise ConnectionError(e, request=request)\\nrequests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\nTo check existing issues, please visit: https://github.com/Azure/azure-cli/issues\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ee5b1a1a3315ac34a62d3e4a95293e1f5e2f2b1ef572b3ff8359f4b3944b63b3","failure_pattern":"requests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve","search_query":"requests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ee5b1a1a3315ac34a62d3e4a95293e1f5e2f2b1ef572b3ff8359f4b3944b63b3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034867837054488576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4477,"pr_state":"closed","pr_sha":"30c786da345c0a04c4820db34dad760cf6200c98","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-20T05:41:12Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"0174a863c6c4d91729700aa5a03e2fc296a3044c9d388d104c12b2ef314b085f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034867837054488576","occurred_at":"2026-03-20T05:41:12Z","failed_at":"provision","signature_id":"55ed0f78d747053c5355d5fabd0e83bc29e9d007dd34869e7c2a094609d6e451","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Frontend/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-20T06:21:29.525Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-20T06:21:30.204Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy values=\"map[adminApi:map[namespace:aro-hcp-admin-api serviceAccount:admin-api] adminApiHost:admin.j4488576.hcpsvc.osadev.cloud apiBindPort:8443 audit:map[connectSocket:false] azure:map[clientId:48b75c14-ea56-4787-9d59-4037fb62f6d9 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[databaseName:arohcpprow-rp-j4488576 databaseUrl:https://arohcpprow-rp-j4488576.documents.azure.com:443/ frontendMiClientId:a16fb73c-60de-4f29-a20d-f30ea1e20ea2 location:westus3] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:frontend-cert-prow-j4488576] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-mjqmwc9t/pipeline@sha256:866c17aa81505a50b1a480b0165400a0a61d5dab459c90451430e1843d948f43 replicas:2 zoneCount:4] exitOnPanic:true mise:map[deploy:false policyLabel:ARM Policy] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-mjqmwc9t/pipeline:pull workloadIdentityClientId:6a650699-8993-4e92-a28a-74e8710f9988 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:frontend workloadIdentityClientId:a16fb73c-60de-4f29-a20d-f30ea1e20ea2 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tracing:map[address:http://ingest.observability:4318 exporter:otlp] virtualService:map[host:rp.j4488576.hcpsvc.osadev.cloud]]\"\ntime=2026-03-20T06:21:30.205Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy\ntime=2026-03-20T06:21:30.548Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-03-20T06:21:30.844Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy dryRun=false\ntime=2026-03-20T06:21:30.844Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy\ntime=2026-03-20T06:21:30.956Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy\ntime=2026-03-20T06:26:40.307Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-frontend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-20T06:26:40.307Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy\ntime=2026-03-20T06:26:40.307Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy\ntime=2026-03-20T06:26:43.871Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy release=aro-hcp-frontend-dev namespace=aro-hcp status=failed description=\"Release \\\"aro-hcp-frontend-dev\\\" failed: resource not ready, name: aro-hcp-frontend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[adminApi:map[namespace:aro-hcp-admin-api serviceAccount:admin-api] adminApiHost:admin.j4488576.hcpsvc.osadev.cloud apiBindPort:8443 audit:map[connectSocket:false] azure:map[clientId:48b75c14-ea56-4787-9d59-4037fb62f6d9 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[databaseName:arohcpprow-rp-j4488576 databaseUrl:https://arohcpprow-rp-j4488576.documents.azure.com:443/ frontendMiClientId:a16fb73c-60de-4f29-a20d-f30ea1e20ea2 location:westus3] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:frontend-cert-prow-j4488576] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-mjqmwc9t/pipeline@sha256:866c17aa81505a50b1a480b0165400a0a61d5dab459c90451430e1843d948f43 replicas:2 zoneCount:4] exitOnPanic:true mise:map[deploy:false policyLabel:ARM Policy] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-mjqmwc9t/pipeline:pull workloadIdentityClientId:6a650699-8993-4e92-a28a-74e8710f9988 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:frontend workloadIdentityClientId:a16fb73c-60de-4f29-a20d-f30ea1e20ea2 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tracing:map[address:http://ingest.observability:4318 exporter:otlp] virtualService:map[host:rp.j4488576.hcpsvc.osadev.cloud]]\"\ntime=2026-03-20T06:26:43.874Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5yUTY/aPhDG7/kUFieQ8Io/+xcHqj3sS9VDtRVSqx5a9eDYQ5jFmYnGEygVH74KLyVQFtE95vHzGz8eTSaLoEYgcS0ekrkzwalTl0focv4CXj8ihXFSQSr6Zit9ciX8kWABtFFS5fxe7n3PjDGm8xm8gE6EFxhAHqNLqdM3nakwKVCwyVfNt5sni0mRLVIh0Jj2vCzQwzMTKsvGKmxnvrL7Ei1tD31F0drFHduGFukSN+HwhEnqSpHpoQ4F6JU37q66955r0vYLz5gfmaZYPLvqqBN+o75e+9+SnHPbElTQpzPUvZdJHeMDUkAqjnJVdYw2PxyckrXOWPCXa3o24Yh+tXHFyEvrQolkXYVW+VLoi0V2qd/ME+vsDdlPxvEv+INTWLpVu9HFVrLwU0HIxYuT/QRV5FUJdO2MHQOHuQFRnKJ3ClZgKpBmIBdvngBI83QgbTBkunZash/vsnmdgxAopPfNj5+ytVnOQMAolpDUlZXJQZcAZLrBKTRytzMcDEd2cGuHgy+D0Xj43/h28K3TMzc35lXPaPx/4+lla/PCSGaOFO6QCKS1rpjMYUu119PpXsrWphJuTg9B+6bisDVfV6RvIhd9U0JKrmhKsgQQk69ajw+Q/O8AAAD//xkTKTBUBQAA\"\ntime=2026-03-20T06:26:43.876Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy\ntime=2026-03-20T06:26:43.876Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-frontend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"42454831adde3c75179282c1ce6d94be508cc22e8880a9bd19abc0e54e5a4d48","failure_pattern":"resource not ready, name: aro-hcp-frontend, kind: Deployment, status: InProgress context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-frontend, kind: Deployment, status: InProgress\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["42454831adde3c75179282c1ce6d94be508cc22e8880a9bd19abc0e54e5a4d48"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4551/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034856757548814336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4551,"pr_state":"closed","pr_sha":"60656aaf62542ef1f0daf794afffe6274be5f47d","final_merged_sha":"60656aaf62542ef1f0daf794afffe6274be5f47d","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T04:57:11Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4535/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034824713200472064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4535,"pr_state":"closed","pr_sha":"3f902fae0079c098423ae1140a498f982a287e3f","final_merged_sha":"3f902fae0079c098423ae1140a498f982a287e3f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T02:49:51Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034823990429618176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4535,"pr_state":"closed","pr_sha":"3f902fae0079c098423ae1140a498f982a287e3f","final_merged_sha":"3f902fae0079c098423ae1140a498f982a287e3f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T02:46:59Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034818141653045248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4403,"pr_state":"open","pr_sha":"19a81cb5753781fff039fc75814c0f5c9b29436a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-20T02:23:39Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034797206417969152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4425,"pr_state":"closed","pr_sha":"855eea85f3052237fc8528c6d280c0ac41ee4307","final_merged_sha":"855eea85f3052237fc8528c6d280c0ac41ee4307","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T01:00:33Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4543/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034789116066926592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4543,"pr_state":"closed","pr_sha":"89c0aa6e9b5745eff17e497b7e08fde82fdde6b5","final_merged_sha":"89c0aa6e9b5745eff17e497b7e08fde82fdde6b5","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-20T00:28:24Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}